
    fThh                     J   S r SSKrSSKJr  SSKJrJrJr  SSKrSSK	rSSKJ
r
  SSKJrJrJr  SSKJrJrJrJr  SS	KJr  SS
KJrJr  SSKJr  \R6                  " \5      r\ " S S\5      5       r " S S\
R>                  5      r  " S S\
R>                  5      r! " S S\
R>                  5      r" " S S\
R>                  5      r# " S S\
R>                  5      r$ " S S\
R>                  5      r% " S S\
R>                  5      r& " S S\
R>                  5      r' " S S \
R>                  5      r( " S! S"\
R>                  5      r) " S# S$\
R>                  5      r*\ " S% S&\5      5       r+\ " S' S(\+5      5       r,\" S)S*9 " S+ S,\+5      5       r-\" S-S*9 " S. S/\+5      5       r./ S0Qr/g)1zPyTorch LeViT model.    N)	dataclass)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionModelOutput)PreTrainedModel)auto_docstringlogging   )LevitConfigc                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                        \	S'   Srg)	,LevitForImageClassificationWithTeacherOutput(   aY  
Output type of [`LevitForImageClassificationWithTeacher`].

Args:
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the `cls_logits` and `distillation_logits`.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
        plus the initial embedding outputs.
Nlogits
cls_logitsdistillation_logitshidden_states )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations__r   r   r   r   __static_attributes__r       `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/levit/modeling_levit.pyr   r   (   sc    $ +/FHU&&'..2J**+27;%"3"34;8<M8E%"3"345<r&   r   c                   8   ^  \ rS rSrSr SU 4S jjrS rSrU =r$ )LevitConvEmbeddingsB   zS
LeViT Conv Embeddings with Batch Norm, used in the initial patch embedding layer.
c	                    > [         T	U ]  5         [        R                  " XX4XVUSS9U l        [        R
                  " U5      U l        g )NF)dilationgroupsbias)super__init__r   Conv2dconvolutionBatchNorm2d
batch_norm)
selfin_channelsout_channelskernel_sizestridepaddingr,   r-   bn_weight_init	__class__s
            r'   r0   LevitConvEmbeddings.__init__G   s@     	99{G_elq
 ..6r&   c                 J    U R                  U5      nU R                  U5      nU$ N)r2   r4   )r5   
embeddingss     r'   forwardLevitConvEmbeddings.forwardP   s&    %%j1
__Z0
r&   )r4   r2   )r   r   r   	r   r   r   r    r!   r0   rA   r%   __classcell__r<   s   @r'   r)   r)   B   s    
 mn7 r&   r)   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )LevitPatchEmbeddingsV   z
LeViT patch embeddings, for final embeddings to be passed to transformer blocks. It consists of multiple
`LevitConvEmbeddings`.
c                 ^  > [         TU ]  5         [        UR                  UR                  S   S-  UR
                  UR                  UR                  5      U l        [        R                  " 5       U l        [        UR                  S   S-  UR                  S   S-  UR
                  UR                  UR                  5      U l        [        R                  " 5       U l        [        UR                  S   S-  UR                  S   S-  UR
                  UR                  UR                  5      U l        [        R                  " 5       U l        [        UR                  S   S-  UR                  S   UR
                  UR                  UR                  5      U l        UR                  U l        g )Nr            )r/   r0   r)   num_channelshidden_sizesr8   r9   r:   embedding_layer_1r   	Hardswishactivation_layer_1embedding_layer_2activation_layer_2embedding_layer_3activation_layer_3embedding_layer_4r5   configr<   s     r'   r0   LevitPatchEmbeddings.__init__\   so   !4!4!4Q!71!<f>P>PRXR_R_agaoao"
 #%,,.!4"a')<)<Q)?1)DfFXFXZ`ZgZgioiwiw"
 #%,,.!4"a')<)<Q)?1)DfFXFXZ`ZgZgioiwiw"
 #%,,.!4"a')<)<Q)?ASASU[UbUbdjdrdr"
 #//r&   c                    UR                   S   nX R                  :w  a  [        S5      eU R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nUR                  S5      R                  SS5      $ )Nr   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rL   )shaperM   
ValueErrorrO   rQ   rR   rS   rT   rU   rV   flatten	transpose)r5   pixel_valuesrM   r@   s       r'   rA   LevitPatchEmbeddings.forwardr   s    #))!,,,,w  ++L9
,,Z8
++J7
,,Z8
++J7
,,Z8
++J7
!!!$..q!44r&   )rQ   rS   rU   rO   rR   rT   rV   rM   rC   rE   s   @r'   rG   rG   V   s    
0,5 5r&   rG   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )MLPLayerWithBN   c                    > [         TU ]  5         [        R                  " XSS9U l        [        R
                  " U5      U l        g )NF)in_featuresout_featuresr.   )r/   r0   r   LinearlinearBatchNorm1dr4   )r5   	input_dim
output_dimr;   r<   s       r'   r0   MLPLayerWithBN.__init__   s1    iiIUZ[..4r&   c                     U R                  U5      nU R                  UR                  SS5      5      R                  U5      nU$ )Nr   r   )rh   r4   r]   
reshape_asr5   hidden_states     r'   rA   MLPLayerWithBN.forward   s<    {{<0|';';Aq'ABMMl[r&   r4   rh   )r   r   r   r   r    r0   rA   r%   rD   rE   s   @r'   rb   rb      s    5
 r&   rb   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )LevitSubsample   c                 :   > [         TU ]  5         Xl        X l        g r?   )r/   r0   r9   
resolution)r5   r9   rx   r<   s      r'   r0   LevitSubsample.__init__   s    $r&   c                     UR                   u  p#nUR                  X R                  U R                  U5      S S 2S S U R                  2S S U R                  24   R	                  USU5      nU$ )N)r[   viewrx   r9   reshape)r5   rp   
batch_size_channelss        r'   rA   LevitSubsample.forward   sg    "."4"4
x#((__dooW_`~$++~~$++~-

'*b(
+ 	 r&   )rx   r9   rs   rE   s   @r'   ru   ru      s    %
 r&   ru   c                   l   ^  \ rS rSrU 4S jr\R                  " 5       SU 4S jj5       rS rS r	Sr
U =r$ )LevitAttention   c                 n  > [         TU ]  5         X0l        US-  U l        X l        X@l        XB-  U-  X#-  S-  -   U l        XB-  U-  U l        [        XR                  5      U l	        [        R                  " 5       U l        [        U R                  USS9U l        [        [        R                   " [#        U5      [#        U5      5      5      n[%        U5      n0 / pU HY  n
U HP  n['        U
S   US   -
  5      ['        U
S   US   -
  5      4nX;  a  [%        U5      X'   U	R)                  X   5        MR     M[     0 U l        [,        R                  R/                  [,        R0                  " U[%        U5      5      5      U l        U R5                  S[,        R6                  " U	5      R9                  Xw5      SS9  g )	N      rL   r   )r;   r   attention_bias_idxsF
persistent)r/   r0   num_attention_headsscalekey_dimattention_ratioout_dim_keys_valuesout_dim_projectionrb   queries_keys_valuesr   rP   
activation
projectionlist	itertoolsproductrangelenabsappendattention_bias_cacher"   	Parameterzerosattention_biasesregister_buffer
LongTensorr|   )r5   rN   r   r   r   rx   points
len_pointsattention_offsetsindicesp1p2offsetr<   s                r'   r0   LevitAttention.__init__   s   #6 d]
.#2#<?R#RU\UruvUv#v "1";>Q"Q#1,@X@X#Y ,,.()@)@,_`ai''j(95;LMN[
%'7Bbebem,c"Q%"Q%-.@A2034E0F%-089	   %'! % 2 25;;?RTWXiTj3k l!5#3#3G#<#A#A*#Yfk 	 	
r&   c                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g r?   r/   trainr   r5   moder<   s     r'   r   LevitAttention.train   )    dD--(*D% .4r&   c                    U R                   (       a  U R                  S S 2U R                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R                  4   U R                  U'   U R                  U   $ r?   trainingr   r   strr   r5   device
device_keys      r'   get_attention_biases#LevitAttention.get_attention_biases   r    ==((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r&   c                    UR                   u  p#nU R                  U5      nUR                  X#U R                  S5      R	                  U R
                  U R
                  U R                  U R
                  -  /SS9u  pgnUR                  SSSS5      nUR                  SSSS5      nUR                  SSSS5      nXgR                  SS5      -  U R                  -  U R                  UR                  5      -   n	U	R                  SS9n	X-  R                  SS5      R                  X#U R                  5      nU R                  U R!                  U5      5      nU$ Nr{   r   dimr   rL   r   )r[   r   r|   r   splitr   r   permuter^   r   r   r   softmaxr}   r   r   r   )
r5   rp   r~   
seq_lengthr   r   querykeyvalue	attentions
             r'   rA   LevitAttention.forward   sB   $0$6$6!
"66|D/44ZTMeMegijpp\\4<<)=)=)LMST q 
E aAq)kk!Q1%aAq)MM"b11DJJ>AZAZ[g[n[nAoo	%%"%-	!)44Q:BB:[_[r[rst|'DEr&   )r   r   r   r   r   r   r   r   r   r   r   Tr   r   r   r    r0   r"   no_gradr   r   rA   r%   rD   rE   s   @r'   r   r      s1    
: ]]_+ +
9 r&   r   c                   l   ^  \ rS rSrU 4S jr\R                  " 5       SU 4S jj5       rS rS r	Sr
U =r$ )LevitAttentionSubsample   c	                 n  > [         TU ]  5         X@l        US-  U l        X0l        XPl        XS-  U-  X4-  -   U l        XS-  U-  U l        Xl        [        XR                  5      U l
        [        Xg5      U l        [        XU-  5      U l        [        R                  " 5       U l        [        U R                  U5      U l        0 U l        ['        [(        R*                  " [-        U5      [-        U5      5      5      n	['        [(        R*                  " [-        U5      [-        U5      5      5      n
[/        U	5      [/        U
5      p0 / pU
 Hv  nU	 Hm  nSn[1        US   U-  US   -
  US-
  S-  -   5      [1        US   U-  US   -
  US-
  S-  -   5      4nUU;  a  [/        U5      UU'   UR3                  UU   5        Mo     Mx     [4        R                  R7                  [4        R8                  " U[/        U5      5      5      U l        U R=                  S[4        R>                  " U5      RA                  X5      SS9  g )Nr   r   r   rL   r   Fr   )!r/   r0   r   r   r   r   r   r   resolution_outrb   keys_valuesru   queries_subsamplequeriesr   rP   r   r   r   r   r   r   r   r   r   r   r"   r   r   r   r   r   r|   )r5   rj   rk   r   r   r   r9   resolution_inr   r   points_r   len_points_r   r   r   r   sizer   r<   s                      r'   r0    LevitAttentionSubsample.__init__   s    	#6 d]
.#2#<?R#RU\Ur#r "1";>Q"Q,))5M5MN!/!F%i;N1NO,,.()@)@*M$&!i''m(<eM>RSTy((~)>n@UVW"%f+s7|K%'7Bbefnr!u4qA~EFBqETZN]_`a]bLbfjmnfnrsesLsHtu!22034E0F%f-089   !& 2 25;;?RTWXiTj3k l!5#3#3G#<#A#A+#Zgl 	 	
r&   c                 f   > [         TU ]  U5        U(       a  U R                  (       a  0 U l        g g g r?   r   r   s     r'   r   LevitAttentionSubsample.train  r   r&   c                    U R                   (       a  U R                  S S 2U R                  4   $ [        U5      nX R                  ;  a*  U R                  S S 2U R                  4   U R                  U'   U R                  U   $ r?   r   r   s      r'   r   ,LevitAttentionSubsample.get_attention_biases  r   r&   c                 4   UR                   u  p#nU R                  U5      R                  X#U R                  S5      R	                  U R
                  U R                  U R
                  -  /SS9u  pVUR                  SSSS5      nUR                  SSSS5      nU R                  U R                  U5      5      nUR                  X R                  S-  U R                  U R
                  5      R                  SSSS5      nXuR                  SS5      -  U R                  -  U R                  UR                  5      -   nUR                  SS9nX-  R                  SS5      R!                  USU R"                  5      nU R%                  U R'                  U5      5      nU$ r   )r[   r   r|   r   r   r   r   r   r   r   r   r^   r   r   r   r   r}   r   r   r   )	r5   rp   r~   r   r   r   r   r   r   s	            r'   rA   LevitAttentionSubsample.forward  sr   $0$6$6!
\*T*$*B*BBGUDLL$"6"6"EFAUN 	
 kk!Q1%aAq)T33LAB

:':':A'=t?W?WY]YeYefnnq!Q
 MM"b11DJJ>AZAZ[g[n[nAoo	%%"%-	!)44Q:BB:rSWSjSjkt|'DEr&   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rE   s   @r'   r   r      s2    +
Z ]]_+ +
9 r&   r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )LevitMLPLayeri-  z=
MLP Layer with `2X` expansion in contrast to ViT with `4X`.
c                    > [         TU ]  5         [        X5      U l        [        R
                  " 5       U l        [        X!5      U l        g r?   )r/   r0   rb   	linear_upr   rP   r   linear_down)r5   rj   
hidden_dimr<   s      r'   r0   LevitMLPLayer.__init__2  s4    '	>,,.)*@r&   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ r?   )r   r   r   ro   s     r'   rA   LevitMLPLayer.forward8  s4    ~~l3|4''5r&   )r   r   r   rC   rE   s   @r'   r   r   -  s    A r&   r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )LevitResidualLayeri?  z
Residual Block for LeViT
c                 :   > [         TU ]  5         Xl        X l        g r?   )r/   r0   module	drop_rate)r5   r   r   r<   s      r'   r0   LevitResidualLayer.__init__D  s    "r&   c                    U R                   (       a  U R                  S:  a  [        R                  " UR	                  S5      SSUR
                  S9nUR                  U R                  5      R                  SU R                  -
  5      R                  5       nXR                  U5      U-  -   nU$ XR                  U5      -   nU$ )Nr   r   )r   )
r   r   r"   randr   r   ge_divdetachr   )r5   rp   rnds      r'   rA   LevitResidualLayer.forwardI  s    ==T^^a/**\..q11a@S@STC''$..)--a$...@AHHJC'++l*Cc*IIL'++l*CCLr&   )r   r   rC   rE   s   @r'   r   r   ?  s    #
   r&   r   c                   8   ^  \ rS rSrSrU 4S jrS rS rSrU =r	$ )
LevitStageiT  zH
LeViT Stage consisting of `LevitMLPLayer` and `LevitAttention` layers.
c                 
  > [         TU ]  5         / U l        Xl        Xl        [        U5       H  nU R                  R                  [        [        X4XgU
5      U R                  R                  5      5        US:  d  MO  X8-  nU R                  R                  [        [        X<5      U R                  R                  5      5        M     U	S   S:X  Ga  U R                  S-
  U	S   -  S-   U l        U R                  R                  [        U R                  R                  X"S-    U	S   U	S   U	S   U	S   U
U R                  S.65        U R                  U l        U	S   S:  a  U R                  R                  US-      U	S   -  nU R                  R                  [        [        U R                  R                  US-      U5      U R                  R                  5      5        [        R                  " U R                  5      U l        g )	Nr   	Subsampler      rL   r   )r   r   r   r9   r   r   rK   )r/   r0   layersrX   r   r   r   r   r   drop_path_rater   r   r   rN   r   
ModuleList)r5   rX   idxrN   r   depthsr   r   	mlp_ratiodown_opsr   r   r   r<   s                r'   r0   LevitStage.__init__Y  s    	*vAKK""<:M`mnKK.. 1})5
""&}\'NPTP[P[PjPjk  A;+%#'#5#5#9hqk"IA"MDKK'[[--c!G<$QK(0$,QK#A;"/#'#6#6
 "&!4!4D{Q![[55cAg>!L
""&%dkk&>&>sQw&GTVZVaVaVpVp mmDKK0r&   c                     U R                   $ r?   )r   )r5   s    r'   get_resolutionLevitStage.get_resolution  s    !!!r&   c                 <    U R                    H  nU" U5      nM     U$ r?   )r   )r5   rp   layers      r'   rA   LevitStage.forward  s     [[E .L !r&   )rX   r   r   r   )
r   r   r   r    r!   r0   r  rA   r%   rD   rE   s   @r'   r   r   T  s    51n" r&   r   c                   6   ^  \ rS rSrSrU 4S jrSS jrSrU =r$ )LevitEncoderi  z;
LeViT Encoder consisting of multiple `LevitStage` stages.
c                   > [         TU ]  5         Xl        U R                  R                  U R                  R                  -  n/ U l        U R                  R                  R                  S/5        [        [        UR                  5      5       H  n[        UUUR                  U   UR                  U   UR                  U   UR                  U   UR                  U   UR                   U   UR                  U   U5
      nUR#                  5       nU R
                  R                  U5        M     [$        R&                  " U R
                  5      U l        g )N )r/   r0   rX   
image_size
patch_sizestagesr   r   r   r   r   r   rN   r   r   r   r   r  r   r   )r5   rX   rx   	stage_idxstager<   s        r'   r0   LevitEncoder.__init__  s   [[++t{{/E/EE
##RD)s6==12I##I.y)i(**95&&y1  +	*E --/JKKu% 3  mmDKK0r&   c                     U(       a  SOS nU R                    H  nU(       a  XA4-   nU" U5      nM     U(       a  XA4-   nU(       d  [        S X4 5       5      $ [        XS9$ )Nr   c              3   .   #    U  H  oc  M  Uv   M     g 7fr?   r   ).0vs     r'   	<genexpr>'LevitEncoder.forward.<locals>.<genexpr>  s     W$Eq$Es   	)last_hidden_stater   )r  tupler   )r5   rp   output_hidden_statesreturn_dictall_hidden_statesr  s         r'   rA   LevitEncoder.forward  sd    "6BD[[E#$5$G! .L !
   1O CW\$EWWW-nnr&   )rX   r  )FTrC   rE   s   @r'   r  r    s    12o or&   r  c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )LevitClassificationLayeri  z
LeViT Classification Layer
c                    > [         TU ]  5         [        R                  " U5      U l        [        R
                  " X5      U l        g r?   )r/   r0   r   ri   r4   rg   rh   )r5   rj   rk   r<   s      r'   r0   !LevitClassificationLayer.__init__  s.    ..3ii	6r&   c                 J    U R                  U5      nU R                  U5      nU$ r?   rr   )r5   rp   r   s      r'   rA    LevitClassificationLayer.forward  s#    |4\*r&   rr   rC   rE   s   @r'   r  r    s    7
 r&   r  c                   ,    \ rS rSr\rSrSrS/rS r	Sr
g)LevitPreTrainedModeli  levitr_   r   c                 *   [        U[        R                  [        R                  45      (       ak  UR                  R
                  R                  SU R                  R                  S9  UR                  b%  UR                  R
                  R                  5         gg[        U[        R                  [        R                  45      (       aJ  UR                  R
                  R                  5         UR                  R
                  R                  S5        gg)zInitialize the weightsg        )meanstdNg      ?)
isinstancer   rg   r1   weightdatanormal_rX   initializer_ranger.   zero_ri   r3   fill_)r5   r   s     r'   _init_weights"LevitPreTrainedModel._init_weights  s    fryy"))455 MM&&CT[[5R5R&S{{&  &&( ' @AAKK""$MM$$S) Br&   r   N)r   r   r   r    r   config_classbase_model_prefixmain_input_name_no_split_modulesr/  r%   r   r&   r'   r#  r#    s     L$O-.
*r&   r#  c                      ^  \ rS rSrU 4S jr\   S	S\\R                     S\\	   S\\	   S\
\\4   4S jj5       rSrU =r$ )

LevitModeli  c                    > [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        U R                  5         g r?   )r/   r0   rX   rG   patch_embeddingsr  encoder	post_initrW   s     r'   r0   LevitModel.__init__  s8      4V <#F+r&   r_   r  r  returnc                 @   Ub  UOU R                   R                  nUb  UOU R                   R                  nUc  [        S5      eU R	                  U5      nU R                  UUUS9nUS   nUR                  SS9nU(       d	  Xg4USS  -   $ [        UUUR                  S9$ )Nz You have to specify pixel_valuesr  r  r   r   r   )r  pooler_outputr   )	rX   r  use_return_dictr\   r8  r9  r&  r   r   )r5   r_   r  r  r@   encoder_outputsr  pooled_outputs           r'   rA   LevitModel.forward  s     %9$D $++JjJj 	 &1%<k$++B]B]?@@**<8
,,!5# ' 
 ,A. *..1.5%58KKK7/')77
 	
r&   )rX   r9  r8  NNN)r   r   r   r    r0   r   r   r"   r#   boolr   r   r   rA   r%   rD   rE   s   @r'   r6  r6    sk      59/3&*	!
u001!
 'tn!
 d^	!

 
u>>	?!
 !
r&   r6  z
    Levit Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    )custom_introc                      ^  \ rS rSrU 4S jr\    S
S\\R                     S\\R                     S\\
   S\\
   S\\\4   4
S jj5       rS	rU =r$ )LevitForImageClassificationi  c                 >  > [         TU ]  U5        Xl        UR                  U l        [	        U5      U l        UR                  S:  a#  [        UR                  S   UR                  5      O[        R                  R                  5       U l        U R                  5         g Nr   r{   )r/   r0   rX   
num_labelsr6  r$  r  rN   r"   r   Identity
classifierr:  rW   s     r'   r0   $LevitForImageClassification.__init__!  s~      ++'

   1$ %V%8%8%<f>O>OP""$ 	 	r&   r_   labelsr  r  r<  c                 .   Ub  UOU R                   R                  nU R                  XUS9nUS   nUR                  S5      nU R	                  U5      nSnUGb  U R                   R
                  c  U R                  S:X  a  SU R                   l        OoU R                  S:  aN  UR                  [        R                  :X  d  UR                  [        R                  :X  a  SU R                   l        OSU R                   l        U R                   R
                  S:X  aI  [        5       n	U R                  S:X  a&  U	" UR                  5       UR                  5       5      nOU	" Xr5      nOU R                   R
                  S:X  a=  [        5       n	U	" UR                  SU R                  5      UR                  S5      5      nO,U R                   R
                  S:X  a  [        5       n	U	" Xr5      nU(       d  U4US	S -   n
Ub  U4U
-   $ U
$ [!        UUUR"                  S
9$ )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr>  r   r   
regressionsingle_label_classificationmulti_label_classificationr{   rL   )lossr   r   )rX   r@  r$  r&  rM  problem_typerK  dtyper"   longintr
   squeezer	   r|   r   r   r   )r5   r_   rO  r  r  outputssequence_outputr   rT  loss_fctoutputs              r'   rA   #LevitForImageClassification.forward1  s    &1%<k$++B]B]**\bm*n!!*)..q11{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE3!//
 	
r&   )rM  rX   r$  rK  )NNNN)r   r   r   r    r0   r   r   r"   r#   r   rE  r   r   r   rA   r%   rD   rE   s   @r'   rH  rH    s       59-1/3&*3
u0013
 ))*3
 'tn	3

 d^3
 
u::	;3
 3
r&   rH  ap  
    LeViT Model transformer with image classification heads on top (a linear layer on top of the final hidden state and
    a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet. .. warning::
           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                      ^  \ rS rSrU 4S jr\   S	S\\R                     S\\	   S\\	   S\
\\4   4S jj5       rSrU =r$ )
&LevitForImageClassificationWithTeacherih  c                   > [         TU ]  U5        Xl        UR                  U l        [	        U5      U l        UR                  S:  a#  [        UR                  S   UR                  5      O[        R                  R                  5       U l        UR                  S:  a#  [        UR                  S   UR                  5      O[        R                  R                  5       U l        U R                  5         g rJ  )r/   r0   rX   rK  r6  r$  r  rN   r"   r   rL  rM  classifier_distillr:  rW   s     r'   r0   /LevitForImageClassificationWithTeacher.__init__q  s      ++'

   1$ %V%8%8%<f>O>OP""$ 	   1$ %V%8%8%<f>O>OP""$ 	 	r&   r_   r  r  r<  c                 &   Ub  UOU R                   R                  nU R                  XUS9nUS   nUR                  S5      nU R	                  U5      U R                  U5      pvXg-   S-  nU(       d  XU4USS  -   n	U	$ [        UUUUR                  S9$ )Nr>  r   r   rL   )r   r   r   r   )rX   r@  r$  r&  rM  rb  r   r   )
r5   r_   r  r  rZ  r[  r   distill_logitsr   r]  s
             r'   rA   .LevitForImageClassificationWithTeacher.forward  s     &1%<k$++B]B]**\bm*n!!*)..q1%)___%EtG^G^_nGoN-2.9GABKGFM;! .!//	
 	
r&   )rM  rb  rX   r$  rK  rD  )r   r   r   r    r0   r   r   r"   r#   rE  r   r   r   rA   r%   rD   rE   s   @r'   r`  r`  h  sk    *  59/3&*	
u001
 'tn
 d^	

 
uBB	C
 
r&   r`  )rH  r`  r6  r#  )0r!   r   dataclassesr   typingr   r   r   r"   torch.utils.checkpointr   torch.nnr   r	   r
   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   configuration_levitr   
get_loggerr   loggerr   Moduler)   rG   rb   ru   r   r   r   r   r   r  r  r#  r6  rH  r`  __all__r   r&   r'   <module>rs     s     ! ) )    A A  . , , 
		H	% =; = =2")) ()5299 )5X	RYY 	RYY ;RYY ;|Pbii PfBII $   *B BJ+o299 +o\ryy   *? * *& +
% +
 +
\ E
"6 E
E
P /
-A /
/
dr&   