o Zh0@sdZddlmZddlmZmZmZmZddlZ ddl m Z ddl m Z mZmZmZddlmZmZmZmZdd lmZmZmZmZmZdd lmZmZdd lm Z d d l!m"Z"e #e$Z%dZ&dZ'gdZ(dZ)dZ*dIdJddZ+Gdddej,j-Z.Gdddej,j-Z/Gdd d ej,j-Z0Gd!d"d"ej,j-Z1Gd#d$d$ej,j-Z2Gd%d&d&ej,j-Z3Gd'd(d(ej,j-Z4Gd)d*d*ej,j-Z5Gd+d,d,ej,j-Z6Gd-d.d.ej,j-Z7Gd/d0d0ej,j-Z8Gd1d2d2ej,j-Z9eGd3d4d4ej,j-Z:Gd5d6d6eZ;d7Zedd>e;eZ?Gd?d@d@ej,j-Z@GdAdBdBej,j-ZAGdCdDdDej,j-ZBedEe<GdFdGdGe;ZCgdHZDdS)KzTensorFlow 2.0 MobileViT model.) annotations)DictOptionalTupleUnionN)get_tf_activation)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardreplace_return_docstrings)TFBaseModelOutputTFBaseModelOutputWithPooling&TFImageClassifierOutputWithNoAttention(TFSemanticSegmenterOutputWithNoAttention)TFPreTrainedModelTFSequenceClassificationLosskeraskeras_serializable unpack_inputs) shape_liststable_softmax)logging)MobileViTConfigrzapple/mobilevit-small)rirztabby, tabby catrvalueintdivisor min_value Optional[int]returncCsF|dur|}t|t||d||}|d|kr||7}t|S)a Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the original TensorFlow repo. It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py Ng?)maxr)rrr new_valuer%b/var/www/auris/lib/python3.10/site-packages/transformers/models/mobilevit/modeling_tf_mobilevit.pymake_divisible@s  r'csBeZdZ      ddfdd Zd d!ddZd"ddZZS)#TFMobileViTConvLayerrFTconfigr in_channelsr out_channels kernel_sizestridegroupsbiasbooldilationuse_normalizationuse_activationUnion[bool, str]r!Nonec stjdi| td|jjdt|dd|} tj | |_ ||dkr6t d|d|dtjj |||d |||d d |_ | rRtjjd d dd|_nd|_| rtt| trbt| |_nt|jtrot|j|_n|j|_nd|_||_||_dS)N z has backpropagation operations that are NOT supported on CPU. If you wish to train/fine-tune this model, you need a GPU or a TPUrr"rzOutput channels (z) are not divisible by z groups.ZVALID convolution)filtersr,stridespaddingZ dilation_rater.use_biasnamegh㈵>g? normalization)epsilonZmomentumr<r%)super__init__loggerwarning __class____name__rrlayersZ ZeroPadding2Dr: ValueErrorZConv2Dr7ZBatchNormalizationr= isinstancestrr activation hidden_actr*r+) selfr)r*r+r,r-r.r/r1r2r3kwargsr:rCr%r&r@Ps<       zTFMobileViTConvLayer.__init__features tf.TensortrainingcCsD||}||}|jdur|j||d}|jdur ||}|SNrP)r:r7r=rI)rKrNrPZpadded_featuresr%r%r&calls     zTFMobileViTConvLayer.callNcCs|jrdSd|_t|dddur2t|jj|jddd|jgWdn1s-wYt|ddduret|j drgt|j j|j ddd|j gWddS1s^wYdSdSdS)NTr7r=r<) builtgetattrtf name_scoper7r<buildr*hasattrr=r+rK input_shaper%r%r&rXs "zTFMobileViTConvLayer.build)rrFrTT)r)rr*rr+rr,rr-rr.rr/r0r1rr2r0r3r4r!r5FrNrOrPr0r!rONrD __module__ __qualname__r@rSrX __classcell__r%r%rMr&r(Os 6 r(cs<eZdZdZ ddfd d ZddddZdddZZS)TFMobileViTInvertedResidualzQ Inverted residual block (MobileNetv2): https://arxiv.org/abs/1801.04381 rr)rr*rr+r-r1r!r5c stjdi|ttt||jd}|dvr!td|d|dko(||k|_t|||ddd|_ t|||d|||d d |_ t|||dd d d |_ dS)Nr)rr"zInvalid stride .r expand_1x1r*r+r,r<rconv_3x3)r*r+r,r-r.r1r<F reduce_1x1r*r+r,r3r<r%) r?r@r'rroundZ expand_ratiorF use_residualr(rergrh)rKr)r*r+r-r1rLZexpanded_channelsrMr%r&r@s4   z$TFMobileViTInvertedResidual.__init__FrNrOrPr0cCs@|}|j||d}|j||d}|j||d}|jr||S|SrQ)rergrhrk)rKrNrPresidualr%r%r&rSs z TFMobileViTInvertedResidual.callNcCs|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|dddurzt|jj|jdWddS1sswYdSdS)NTrergrh) rTrUrVrWrer<rXrgrhrZr%r%r&rXs "z!TFMobileViTInvertedResidual.buildr) r)rr*rr+rr-rr1rr!r5r\r]r^rDr`ra__doc__r@rSrXrbr%r%rMr&rcs  # rccs:eZdZ  ddfd d ZddddZdddZZS)TFMobileViTMobileNetLayerrr)rr*rr+r- num_stagesr!r5c  s^tjdi|g|_t|D]}t||||dkr|ndd|d}|j||}qdS)Nrrlayer.)r*r+r-r<r%)r?r@rErangercappend) rKr)r*r+r-rqrLilayerrMr%r&r@s   z"TFMobileViTMobileNetLayer.__init__FrNrOrPr0cC|jD]}|||d}q|SrQrE)rKrNrP layer_moduler%r%r&rS zTFMobileViTMobileNetLayer.callNc Cj|jrdSd|_t|dddur1|jD]}t|j |dWdn1s+wYqdSdSNTrErTrUrErVrWr<rXrKr[ryr%r%r&rX  zTFMobileViTMobileNetLayer.build)rr) r)rr*rr+rr-rrqrr!r5r\r]r^r_r%r%rMr&rps  rpcs>eZdZdfdd Zdd d ZddddZdddZZS)TFMobileViTSelfAttentionr)r hidden_sizerr!r5c stjd i|||jdkrtd|d|jd|j|_t||j|_|j|j|_tj|jtj d}tj ||_ t jj|j|jdd|_t jj|j|jdd|_t jj|j|jd d|_t j|j|_||_dS) NrzThe hidden size z4 is not a multiple of the number of attention heads rddtypequery)r;r<keyrr%)r?r@num_attention_headsrFrattention_head_size all_head_sizerVcastZfloat32mathsqrtscalerrEDenseZqkv_biasrrrDropoutZattention_probs_dropout_probdropoutr)rKr)rrLrrMr%r&r@s" z!TFMobileViTSelfAttention.__init__xrOcCs:t|d}tj||d|j|jfd}tj|gddS)Nrshaperr"rrperm)rVrreshaperr transpose)rKr batch_sizer%r%r&transpose_for_scoressz-TFMobileViTSelfAttention.transpose_for_scoresF hidden_statesrPr0c Cst|d}|||}|||}|||}tj||dd}||j}t|dd}|j ||d}t||} tj | gdd} tj | |d|j fd } | S) NrT)Z transpose_brZaxisrRrrr) rVrrrrrmatmulrrrrrr) rKrrPrZ key_layerZ value_layerZ query_layerZattention_scoresZattention_probsZ context_layerr%r%r&rSs   zTFMobileViTSelfAttention.callNcCs|jrdSd|_t|dddur1t|jj|jdd|jgWdn1s,wYt|dddurZt|jj|jdd|jgWdn1sUwYt|dddurt|j j|j dd|jgWddS1swYdSdS)NTrrr) rTrUrVrWrr<rXrrrrZr%r%r&rX7s "zTFMobileViTSelfAttention.buildr)rrrr!r5)rrOr!rOr\rrOrPr0r!rOr^)rDr`rar@rrSrXrbr%r%rMr&rs   rcs4eZdZdfdd ZddddZdddZZS)TFMobileViTSelfOutputr)rrrr!r5c s>tjdi|tjj|dd|_tj|j|_||_ dSNdenser<r%) r?r@rrErrrhidden_dropout_probrrrKr)rrLrMr%r&r@G zTFMobileViTSelfOutput.__init__FrrOrPr0cCs||}|j||d}|SrQrr)rKrrPr%r%r&rSMrzzTFMobileViTSelfOutput.callNcCl|jrdSd|_t|dddur4t|jj|jdd|jgWddS1s-wYdSdSNTrrTrUrVrWrr<rXrrZr%r%r&rXR"zTFMobileViTSelfOutput.buildrr\rr^r_r%r%rMr&rFs rcs<eZdZdfdd Zd d ZddddZdddZZS)TFMobileViTAttentionr)rrrr!r5c s6tjdi|t||dd|_t||dd|_dS)N attentionroutputr%)r?r@rrr dense_outputrrMr%r&r@\szTFMobileViTAttention.__init__cCtr^NotImplementedError)rKZheadsr%r%r& prune_headsasz TFMobileViTAttention.prune_headsFrrOrPr0cCs |j||d}|j||d}|SrQ)rr)rKrrPZ self_outputsattention_outputr%r%r&rSdszTFMobileViTAttention.callNcC|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurUt|jj|jdWddS1sNwYdSdS)NTrr)rTrUrVrWrr<rXrrZr%r%r&rXi"zTFMobileViTAttention.buildrr\rr^)rDr`rar@rrSrXrbr%r%rMr&r[s  rcs2eZdZdfdd Zdd d ZdddZZS)TFMobileViTIntermediater)rrrintermediate_sizer!r5c sPtjdi|tjj|dd|_t|jtrt |j|_ n|j|_ ||_ dSr) r?r@rrErrrGrJrHrintermediate_act_fnrrKr)rrrLrMr%r&r@vs   z TFMobileViTIntermediate.__init__rrOcCs||}||}|Sr^)rr)rKrr%r%r&rSs  zTFMobileViTIntermediate.callNcCrrrrZr%r%r&rXrzTFMobileViTIntermediate.buildr)rrrrrr!r5)rrOr!rOr^r_r%r%rMr&rus rcs4eZdZdfdd ZddddZdddZZS)TFMobileViTOutputr)rrrrr!r5c s>tjdi|tjj|dd|_tj|j|_||_ dSr) r?r@rrErrrrrrrrMr%r&r@rzTFMobileViTOutput.__init__FrrO input_tensorrPr0cCs$||}|j||d}||}|SrQr)rKrrrPr%r%r&rSs zTFMobileViTOutput.callNcCrr)rTrUrVrWrr<rXrrZr%r%r&rXrzTFMobileViTOutput.buildrr\)rrOrrOrPr0r!rOr^r_r%r%rMr&rs rc4eZdZdfdd ZddddZdddZZS)TFMobileViTTransformerLayerr)rrrrr!r5c sxtjdi|t||dd|_t|||dd|_t|||dd|_tj j |j dd|_ tj j |j dd|_ ||_dS) Nrr intermediaterlayernorm_beforer>r<layernorm_afterr%)r?r@rrrrrmobilevit_outputrrELayerNormalizationlayer_norm_epsrrrrrMr%r&r@s z$TFMobileViTTransformerLayer.__init__FrrOrPr0cCsD|j|||d}||}||}||}|j|||d}|SrQ)rrrrr)rKrrPrZ layer_outputr%r%r&rSs   z TFMobileViTTransformerLayer.callNcCs|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|dddurwt|jj|jdWdn1srwYt|dddurt|j j|j dd|j gWdn1swYt|dddurt|j j|j dd|j gWddS1swYdSdS)NTrrrrr) rTrUrVrWrr<rXrrrrrrZr%r%r&rXs0"z!TFMobileViTTransformerLayer.buildrr\rr^r_r%r%rMr&rs  rcr)TFMobileViTTransformerr)rrrrqr!r5c sVtjdi|g|_t|D]}t||t||jd|d}|j|qdS)Nrr)rrr<r%)r?r@rErsrrZ mlp_ratiort)rKr)rrqrLruZtransformer_layerrMr%r&r@s  zTFMobileViTTransformer.__init__FrrOrPr0cCrwrQrx)rKrrPryr%r%r&rSrzzTFMobileViTTransformer.callNc Cr{r|r}r~r%r%r&rXrzTFMobileViTTransformer.build)r)rrrrqrr!r5r\rr^r_r%r%rMr&rs rcsPeZdZdZ d"d#fdd Zd$ddZd%ddZd&d'ddZd(d d!ZZ S))TFMobileViTLayerz; MobileViT block: https://arxiv.org/abs/2110.02178 rr)rr*rr+r-rrqr1r!r5c stjdi||j|_|j|_|dkr1t||||dkr|nd|dkr(|dnddd|_|}nd|_t||||jdd|_ t|||ddddd |_ t |||d d |_ t jj|jd d |_t|||ddd|_t|d|||jdd|_||_dS)Nr"rdownsampling_layer)r*r+r-r1r<conv_kxkrfFconv_1x1)r*r+r,r2r3r< transformer)rrqr< layernormrconv_projectionfusionr%)r?r@Z patch_size patch_width patch_heightrcrr(Zconv_kernel_sizerrrrrrErrrrrr) rKr)r*r+r-rrqr1rLrMr%r&r@sZ    zTFMobileViTLayer.__init__rNrOTuple[tf.Tensor, Dict]cCsf|j|j}}t||d}t|d}t|d}t|d}t|d}ttj|||d} ttj|||d} | |kpN| |k} | r\tjj|| | fdd}| |} | |} | | }t |gd}t |||| || |f}t |gd }t |||||f}t |gd }t |||||f}||f||| || | d }||fS) NZint32rrr"rbilinearsizemethodrrrr"rrrr"r) orig_sizerchannels interpolate num_patchesnum_patches_widthnum_patches_height) rrrVrrrceilimageresizerr)rKrNrr patch_arearZ orig_heightZ orig_widthrZ new_heightZ new_widthrnum_patch_widthnum_patch_heightrpatches info_dictr%r%r& unfolding.s> zTFMobileViTLayer.unfoldingrrrc Cs|j|j}}t||}|d}|d}|d}|d} |d} t||||df} tj| dd} t| ||| | ||f} tj| d d} t| ||| || |f} tj| d d} |d rktjj| |d d d} | S)Nrrrrrrrrrrr"rrrrrr)rrrrVrrrr) rKrrrrrrrrrrrNr%r%r&foldingZs( zTFMobileViTLayer.foldingFrPr0cCs|jr |j||d}|}|j||d}|j||d}||\}}|j||d}||}|||}|j||d}|jt j ||gdd|d}|S)NrRrr) rrrrrrrrrrVconcat)rKrNrPrlrrr%r%r&rSvs  zTFMobileViTLayer.callNcCs,|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|dddurwt|jj|jdWdn1srwYt|dddurt|j j|j dd|j gWdn1swYt|dddurt|j j|j dWdn1swYt|dddurt|j j|j dWdn1swYt|dddurt|j j|j dWddS1s wYdSdS) NTrrrrrrr)rTrUrVrWrr<rXrrrrrrrrZr%r%r&rXs@$zTFMobileViTLayer.buildrm)r)rr*rr+rr-rrrrqrr1rr!r5)rNrOr!r)rrOrrr!rOr\r]r^) rDr`raror@rrrSrXrbr%r%rMr&rs  A , rcs:eZdZdfdd Z   ddddZdddZZS)TFMobileViTEncoderr)rr!r5c s`tjdi|||_g|_d}}|jdkrd}d}n|jdkr$d}d}t||jd|jddddd}|j|t||jd|jd d d d d}|j|t||jd |jd d |j dd d d }|j||rr|d 9}t||jd |jdd |j dd|dd} |j| |r|d 9}t||jd|jdd |j d d |dd} |j| dS)NFrTrrzlayer.0)r*r+r-rqr<r"rzlayer.1zlayer.2)r*r+r-rrqr<zlayer.3)r*r+r-rrqr1r<zlayer.4r%) r?r@r)rEZ output_striderpneck_hidden_sizesrtrZ hidden_sizes) rKr)rLZdilate_layer_4Zdilate_layer_5r1Zlayer_1Zlayer_2Zlayer_3Zlayer_4Zlayer_5rMr%r&r@s       zTFMobileViTEncoder.__init__FTrrOoutput_hidden_statesr0 return_dictrPUnion[tuple, TFBaseModelOutput]cCs`|rdnd}t|jD]\}}|||d}|r||f}q |s*tdd||fDSt||dS)Nr%rRcss|] }|dur|VqdSr^r%).0vr%r%r& sz*TFMobileViTEncoder.call..)last_hidden_stater) enumeraterEtupler )rKrrrrPZall_hidden_statesruryr%r%r&rSs    zTFMobileViTEncoder.callNc Cr{r|r}r~r%r%r&rX rzTFMobileViTEncoder.buildr)rr!r5)FTF) rrOrr0rr0rPr0r!rr^r_r%r%rMr&rsQ rcsNeZdZeZddfdd Zdd Ze ddddZdddZ Z S)TFMobileViTMainLayerTr)r expand_outputr0c stjdi|||_||_t||j|jddddd|_t|dd|_ |jr8t||jd|jd d d d |_ t j j d dd|_dS)Nrrr" conv_stem)r*r+r,r-r<encoderrrr conv_1x1_exprfZchannels_firstpooler)Z data_formatr<r%)r?r@r)rr(Z num_channelsrrrrrrrEGlobalAveragePooling2Dr)rKr)rrLrMr%r&r@s* zTFMobileViTMainLayer.__init__cCr)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel r)rKZheads_to_pruner%r%r& _prune_heads6sz!TFMobileViTMainLayer._prune_headsNF pixel_valuestf.Tensor | NonerOptional[bool]rrPr!5Union[Tuple[tf.Tensor], TFBaseModelOutputWithPooling]c Cs<|dur|n|jj}|dur|n|jj}tj|dd}|j||d}|j||||d}|jrD||d}tj|gdd}| |}n|d}tj|gdd}d}|s|dur]||fn|f} |js{|dd} t dd | dD} | f} | | S| |ddS|rt d d |dD} t |||r| d S|j d S) NrrrRrrrPrrrcSg|] }tj|ddqSrrrVrrhr%r%r& jz-TFMobileViTMainLayer.call..cSrr r r r%r%r&r sr)r pooler_outputr) r)ruse_return_dictrVrrrrrrrrr) rKrrrrPZembedding_outputZencoder_outputsr pooled_outputrZremaining_encoder_outputsrr%r%r&rS=sD   zTFMobileViTMainLayer.callcCsF|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|ddduryt|jj|jgdWdn1stwYt|dddurt|j j|j dWddS1swYdSdS)NTrrrNNNNr) rTrUrVrWrr<rXrrrrZr%r%r&rX{s("zTFMobileViTMainLayer.buildTr)rrr0NNNF rrrrrrrPr0r!rr^) rDr`rar config_classr@rrrSrXrbr%r%rMr&rs=rc@seZdZdZeZdZdZdS)TFMobileViTPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mobilevitrN)rDr`rarorrZbase_model_prefixZmain_input_namer%r%r%r&rs ra This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `pixel_values` only and nothing else: `model(pixel_values)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([pixel_values, attention_mask])` or `model([pixel_values, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"pixel_values": pixel_values, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`MobileViTConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. a Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]`, `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. zWThe bare MobileViT model outputting raw hidden-states without any specific head on top.c s\eZdZddfdd Zeeeeee e de d  ddddZ dddZ ZS)TFMobileViTModelTr)rrr0cs:tj|g|Ri|||_||_t||dd|_dS)Nrrr<)r?r@r)rrr)rKr)rinputsrLrMr%r&r@szTFMobileViTModel.__init__Zvision) checkpoint output_typerZmodalityexpected_outputNFrrrrrrPr!rcCs|j||||d}|SrQ)r)rKrrrrPrr%r%r&rSszTFMobileViTModel.callcCsd|jrdSd|_t|dddur0t|jj|jdWddS1s)wYdSdS)NTr)rTrUrVrWrr<rXrZr%r%r&rXs"zTFMobileViTModel.buildrrrrr^)rDr`rar@rr MOBILEVIT_INPUTS_DOCSTRINGr _CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErSrXrbr%r%rMr&rs"  rz MobileViT model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. csZeZdZdfdd Zeeeeee e e d     ddddZ dddZ ZS)!TFMobileViTForImageClassificationr)rr!r5csntj|g|Ri||j|_t|dd|_tj|j|_ |jdkr.tjj |jddnt j |_ ||_dS)Nrrr classifier)r?r@ num_labelsrrrrErclassifier_dropout_probrrrVidentityr%r))rKr)rrLrMr%r&r@s  z*TFMobileViTForImageClassification.__init__)rrrrNFrrrrlabelsrrP4Union[tuple, TFImageClassifierOutputWithNoAttention]c Cs|dur|n|jj}|j||||d}|r|jn|d}||j||d}|dur,dn|j||d} |sI|f|dd} | durG| f| S| St| ||jdS)a labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NrrrR)r)logitsr"lossr+r) r)rrrr%rhf_compute_lossrr) rKrrr)rrPoutputsrr+r-rr%r%r&rSsz&TFMobileViTForImageClassification.callcCs|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurbt|jdrdt|jj|jdd|j j dgWddS1s[wYdSdSdS)NTrr%r<r) rTrUrVrWrr<rXrYr%r)rrZr%r%r&rX5s "z'TFMobileViTForImageClassification.buildrNNNNF) rrrrr)rrrrPrr!r*r^)rDr`rar@rr r r _IMAGE_CLASS_CHECKPOINTrr"_IMAGE_CLASS_EXPECTED_OUTPUTrSrXrbr%r%rMr&r$s" r$cr)TFMobileViTASPPPoolingr)rr*rr+r!r5c sBtjdi|tjjddd|_t|||dddddd|_dS) NT global_pool)Zkeepdimsr<rrelur)r*r+r,r-r2r3r<r%)r?r@rrErr4r(r)rKr)r*r+rLrMr%r&r@Cs zTFMobileViTASPPPooling.__init__FrNrOrPr0cCs>t|dd}||}|j||d}tjj||dd}|S)NrrrRrr)rr4rrVrr)rKrNrPZ spatial_sizer%r%r&rSSs  zTFMobileViTASPPPooling.callNcCs|jrdSd|_t|dddur/t|jj|jgdWdn1s*wYt|dddurWt|jj|jdWddS1sPwYdSdS)NTr4rr)rTrUrVrWr4r<rXrrZr%r%r&rXZs"zTFMobileViTASPPPooling.build)r)rr*rr+rr!r5r\r]r^r_r%r%rMr&r3Bs r3c8eZdZdZdfdd Zddd dZdddZZS)TFMobileViTASPPzs ASPP module defined in DeepLab papers: https://arxiv.org/abs/1606.00915, https://arxiv.org/abs/1706.05587 r)rr!r5c stjdi|jdjtjdkrtdg|_tdddd}|j ||j fdd t jDt d tjdd }|j |td ddd d|_ tjj|_dS)Nrz"Expected 3 values for atrous_ratesrr5zconvs.0ric s0g|]\}}td|dd|ddqS)rr5convs.r)r*r+r,r1r3r<)r()rruZrater)r*r+r%r&r s  z,TFMobileViTASPP.__init__..r9rrprojectr%)r?r@raspp_out_channelslenZ atrous_ratesrFconvsr(rtextendrr3r;rrErZaspp_dropout_probr)rKr)rLZ in_projectionZ pool_layerrMr:r&r@ksB     zTFMobileViTASPP.__init__FrNrOrPr0cCsbtj|gdd}g}|jD] }||||dqtj|dd}|j||d}|j||d}|S)NrrrRrr)rVrr>rtrr;r)rKrNrPZpyramidconvZpooled_featuresr%r%r&rSs zTFMobileViTASPP.callNc Cs|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurV|jD]}t|j |dWdn1sPwYq8dSdS)NTr;r>)rTrUrVrWr;r<rXr>)rKr[r@r%r%r&rXs  zTFMobileViTASPP.buildrr\r]r^rnr%r%rMr&r7fs  4 r7cr6)TFMobileViTDeepLabV3zB DeepLabv3 architecture: https://arxiv.org/abs/1706.05587 r)rr!r5c sRtjdi|t|dd|_tj|j|_t ||j |j dddddd|_ dS) NaspprrFTr%)r*r+r,r2r3r/r<r%) r?r@r7rBrrErr'rr(r<r&r%rKr)rLrMr%r&r@s zTFMobileViTDeepLabV3.__init__FrrOrPr0cCs2|j|d|d}|j||d}|j||d}|S)NrrR)rBrr%)rKrrPrNr%r%r&rSszTFMobileViTDeepLabV3.callNcCr)NTrBr%)rTrUrVrWrBr<rXr%rZr%r%r&rXrzTFMobileViTDeepLabV3.buildrr\rr^rnr%r%rMr&rAs  rAzX MobileViT model with a semantic segmentation head on top, e.g. for Pascal VOC. cs^eZdZdfdd ZddZeeeee e d  ddddZ dddZ Z S)"TFMobileViTForSemanticSegmentationr)rr!r5c s>tj|fi||j|_t|ddd|_t|dd|_dS)NFrrsegmentation_headr)r?r@r&rrrArErCrMr%r&r@sz+TFMobileViTForSemanticSegmentation.__init__csJt|dd}tjj||dd}tjjdddfdd}|||S) NrrrTnone)Z from_logitsZ reductioncsJ||}tj|jjk|jd}||}t|t|}t|dS)Nrrm)rVrr)Zsemantic_loss_ignore_indexrZ reduce_sumr)realpredZ unmasked_lossmask masked_lossZreduced_masked_lossZloss_fctrKr%r&rJs  zGTFMobileViTForSemanticSegmentation.hf_compute_loss..masked_loss)rrVrrrZlossesZSparseCategoricalCrossentropy)rKr+r)Zlabel_interp_shapeZupsampled_logitsrJr%rKr&r.s  z2TFMobileViTForSemanticSegmentation.hf_compute_loss)rrNFrrr)rrrrPr06Union[tuple, TFSemanticSegmenterOutputWithNoAttention]c Cs|dur|n|jj}|dur|n|jj}|dur"|jjdks"td|j|d||d}|r0|jn|d}|j||d}d} |durH|j||d} t j |gdd }|ss|r_|f|dd} n |f|d d} | durq| f| S| St | ||r}|jd Sdd S) aK labels (`tf.Tensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> from transformers import AutoImageProcessor, TFMobileViTForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") >>> model = TFMobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small") >>> inputs = image_processor(images=image, return_tensors="tf") >>> outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```Nrz/The number of labels should be greater than oneTrrR)r+r)rrr"r,) r)rrr&rFrrrEr.rVrr) rKrr)rrrPr/Zencoder_hidden_statesr+r-rr%r%r&rSs:'z'TFMobileViTForSemanticSegmentation.callcCr)NTrrE)rTrUrVrWrr<rXrErZr%r%r&rXQrz(TFMobileViTForSemanticSegmentation.buildrr0) rrr)rrrrrrPr0r!rLr^)rDr`rar@r.rr r r rr"rSrXrbr%r%rMr&rDs KrD)r$rDrr)rN)rrrrrr r!r)Ero __future__rtypingrrrrZ tensorflowrVZactivations_tfrZ file_utilsr r r r Zmodeling_tf_outputsr rrrZmodeling_tf_utilsrrrrrZtf_utilsrrutilsrZconfiguration_mobilevitrZ get_loggerrDrAr"r!r#r1r2r'rEZLayerr(rcrprrrrrrrrrrrZMOBILEVIT_START_DOCSTRINGr rr$r3r7rArD__all__r%r%r%r&sn      M@'C(Cmu )$B$S( v