
    fThQ                        S r SSKrSSKJr  SSKJrJrJrJ	r	J
r
  SSKrSSKrSSKJr  SSKJrJrJr  SSKJr  SS	KJrJrJrJr  SS
KJrJr  SSKJrJr  SSKJ r J!r!J"r"J#r#  SSK$J%r%  \"RL                  " \'5      r( " S S\RR                  5      r* " S S\RR                  5      r+ S<S\RR                  S\RX                  S\RX                  S\RX                  S\\RX                     S\-S\-4S jjr. " S S\RR                  5      r/ " S S\RR                  5      r0 " S S \RR                  5      r1 " S! S"\RR                  5      r2 " S# S$\RR                  5      r3 " S% S&\RR                  5      r4 " S' S(\RR                  5      r5\! " S) S*\5      5       r6\! " S+ S,\65      5       r7 " S- S.\RR                  5      r8\!" S/S09 " S1 S2\65      5       r9\!" S3S09 " S4 S5\65      5       r:\ " S6 S7\ 5      5       r;\!" S8S09 " S9 S:\65      5       r</ S;Qr=g)=zPyTorch DeiT model.    N)	dataclass)CallableOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputauto_docstringlogging	torch_int   )
DeiTConfigc            	          ^  \ rS rSrSrSS\S\SS4U 4S jjjrS\R                  S	\
S
\
S\R                  4S jr  SS\R                  S\\R                     S\S\R                  4S jjrSrU =r$ )DeiTEmbeddings*   zn
Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
configuse_mask_tokenreturnNc                   > [         TU ]  5         [        R                  " [        R
                  " SSUR                  5      5      U l        [        R                  " [        R
                  " SSUR                  5      5      U l        U(       a6  [        R                  " [        R
                  " SSUR                  5      5      OS U l	        [        U5      U l        U R                  R                  n[        R                  " [        R
                  " SUS-   UR                  5      5      U l        [        R                  " UR                  5      U l        UR"                  U l        g )Nr      )super__init__r	   	Parametertorchzeroshidden_size	cls_tokendistillation_token
mask_tokenDeiTPatchEmbeddingspatch_embeddingsnum_patchesposition_embeddingsDropouthidden_dropout_probdropout
patch_size)selfr    r!   r0   	__class__s       ^/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/deit/modeling_deit.pyr&   DeiTEmbeddings.__init__/   s    ekk!Q8J8J&KL"$,,u{{1aASAS/T"UQ_",,u{{1a9K9K'LMei 3F ;++77#%<<A{QPVPbPb0c#d zz&"<"<= ++    
embeddingsheightwidthc                    UR                   S   S-
  nU R                  R                   S   S-
  n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  SS2SS24   nU R                  SS2SS24   nUR                   S   nX R
                  -  n	X0R
                  -  n
[        US-  5      nUR                  SXU5      nUR                  SSSS5      n[        R                  R                  UX4SS	S
9nUR                  SSSS5      R                  SSU5      n[        R                  " Xg4SS9$ )a  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing and 2 class embeddings.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   r$   N      ?r   r   bicubicF)sizemodealign_cornersdim)shaper1   r(   jit
is_tracingr5   r   reshapepermuter	   
functionalinterpolateviewcat)r6   r;   r<   r=   r0   num_positionsclass_and_dist_pos_embedpatch_pos_embedrF   
new_height	new_widthsqrt_num_positionss               r8   interpolate_pos_encoding'DeiTEmbeddings.interpolate_pos_encoding;   sU    !&&q)A-0066q9A= yy##%%+*F6?+++#'#;#;ArrE#B 221ab59r".
__,	&}c'9:)11!5G]`a)11!Q1=--33(	 4 
 *11!Q1=BB1b#Nyy2D!LLr:   pixel_valuesbool_masked_posrV   c                    UR                   u    pEnU R                  U5      nUR                  5       u  pnUbI  U R                  R	                  XS5      n
UR                  S5      R                  U
5      nUSU-
  -  X-  -   nU R                  R	                  USS5      nU R                  R	                  USS5      n[        R                  " XU4SS9nU R                  nU(       a  U R                  XuU5      nX~-   nU R                  U5      nU$ )Nr?         ?r   rE   )rG   r/   rB   r-   expand	unsqueezetype_asr+   r,   r(   rO   r1   rV   r4   )r6   rX   rY   rV   _r<   r=   r;   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokensposition_embeddings                  r8   forwardDeiTEmbeddings.forwardc   s    +001e**<8
$.OO$5!
&//00LK",,R088ED#sTz2[5GGJ^^**:r2>
"55<<ZRPYY
LRST
!55#!%!>!>zSX!Y4
\\*-
r:   )r+   r,   r4   r-   r/   r5   r1   )FNF)__name__
__module____qualname____firstlineno____doc__r   boolr&   r(   TensorintrV   r   
BoolTensorrg   __static_attributes____classcell__r7   s   @r8   r   r   *   s    
,z 
,4 
,D 
, 
,&M5<< &M &MUX &M]b]i]i &MV 7;).	ll "%"2"23 #'	
 
 r:   r   c                   f   ^  \ rS rSrSrU 4S jrS\R                  S\R                  4S jrSr	U =r
$ )r.      z
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                   > [         TU ]  5         UR                  UR                  p2UR                  UR
                  pT[        U[        R                  R                  5      (       a  UOX"4n[        U[        R                  R                  5      (       a  UOX34nUS   US   -  US   US   -  -  nX l        X0l        X@l        X`l
        [        R                  " XEX3S9U l        g )Nr   r   )kernel_sizestride)r%   r&   
image_sizer5   num_channelsr*   
isinstancecollectionsabcIterabler0   r	   Conv2d
projection)r6   r    r{   r5   r|   r*   r0   r7   s          r8   r&   DeiTPatchEmbeddings.__init__   s    !'!2!2F4E4EJ$*$7$79K9Kk#-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
15*Q-:VW=:XY$$(&))L:ir:   rX   r"   c                     UR                   u  p#pEX0R                  :w  a  [        S5      eU R                  U5      R	                  S5      R                  SS5      nU$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r$   r   )rG   r|   
ValueErrorr   flatten	transpose)r6   rX   r`   r|   r<   r=   xs          r8   rg   DeiTPatchEmbeddings.forward   s[    2>2D2D/
&,,,w  OOL)11!4>>q!Dr:   )r{   r|   r0   r5   r   )rj   rk   rl   rm   rn   r&   r(   rp   rg   rs   rt   ru   s   @r8   r.   r.      s.    jELL U\\  r:   r.   modulequerykeyvalueattention_maskscalingr4   c                    [         R                  " XR                  SS5      5      U-  n[        R                  R                  US[         R                  S9R                  UR                  5      n[        R                  R                  XU R                  S9nUb  X-  n[         R                  " X5      n	U	R                  SS5      R                  5       n	X4$ )Nr?   )rF   dtype)ptrainingr   r$   )r(   matmulr   r	   rL   softmaxfloat32tor   r4   r   
contiguous)
r   r   r   r   r   r   r4   kwargsattn_weightsattn_outputs
             r8   eager_attention_forwardr      s     <<}}R'<=GL ==((2U]](SVVW\WbWbcL ==((6??([L !#4,,|3K''1-88:K$$r:   c            
          ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jr SS\	\R                     S	\
S\\\R                  \R                  4   \\R                     4   4S
 jjrSrU =r$ )DeiTSelfAttention   r    r"   Nc                 0  > [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eXl        UR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        UR                  U l        U R                  S-  U l        SU l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        g )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .g      F)bias)r%   r&   r*   num_attention_headshasattrr   r    rq   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr	   Linearqkv_biasr   r   r   r6   r    r7   s     r8   r&   DeiTSelfAttention.__init__   sG    : ::a?PVXhHiHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r:   r   c                     UR                  5       S S U R                  U R                  4-   nUR                  U5      nUR	                  SSSS5      $ )Nr?   r   r$   r   r   )rB   r   r   rN   rK   )r6   r   new_x_shapes      r8   transpose_for_scores&DeiTSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r:   	head_maskoutput_attentionsc                    U R                  U R                  U5      5      nU R                  U R                  U5      5      nU R                  U R                  U5      5      n[        nU R
                  R                  S:w  aT  U R
                  R                  S:X  a  U(       a  [        R                  S5        O[        U R
                  R                     nU" U UUUUU R                  U R                  U R                  (       d  SOU R                  S9u  pUR                  5       S S U R                  4-   n
UR!                  U
5      nU(       a  X4nU$ U4nU$ )Neagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        )r   r   r4   r   )r   r   r   r   r   r    _attn_implementationloggerwarning_oncer   r   r   r   r   rB   r   rJ   )r6   hidden_statesr   r   	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapeoutputss               r8   rg   DeiTSelfAttention.forward   s9    --dhh}.EF	//

=0IJ//

=0IJ(?;;++w6{{//69>O##L
 '>dkk>^>^&_#)<nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EF6G=2 O\M]r:   )
r   r   r    r   r   r   r   r   r   r   ri   )rj   rk   rl   rm   r   r&   r(   rp   r   r   ro   r   r   rg   rs   rt   ru   s   @r8   r   r      s    ]z ]d ](%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F! !r:   r   c                      ^  \ rS rSrSrS\SS4U 4S jjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )DeiTSelfOutputi  z
The residual connection is defined in DeiTLayer instead of here (as is the case with other models), due to the
layernorm applied before each block.
r    r"   Nc                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g N)	r%   r&   r	   r   r*   denser2   r3   r4   r   s     r8   r&   DeiTSelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r:   r   input_tensorc                 J    U R                  U5      nU R                  U5      nU$ r   r   r4   r6   r   r   s      r8   rg   DeiTSelfOutput.forward  s$    

=1]3r:   r   )rj   rk   rl   rm   rn   r   r&   r(   rp   rg   rs   rt   ru   s   @r8   r   r     sI    
>z >d >
U\\  RWR^R^  r:   r   c                      ^  \ rS rSrS\SS4U 4S jjrS\\   SS4S jr  SS\	R                  S	\\	R                     S
\S\\\	R                  \	R                  4   \\	R                     4   4S jjrSrU =r$ )DeiTAttentioni  r    r"   Nc                    > [         TU ]  5         [        U5      U l        [	        U5      U l        [        5       U l        g r   )r%   r&   r   	attentionr   outputsetpruned_headsr   s     r8   r&   DeiTAttention.__init__  s0    *62$V,Er:   headsc                 6   [        U5      S:X  a  g [        XR                  R                  U R                  R                  U R
                  5      u  p[        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l	        [        U R                  R                  USS9U R                  l        U R                  R                  [        U5      -
  U R                  l        U R                  R                  U R                  R                  -  U R                  l        U R
                  R                  U5      U l        g )Nr   r   rE   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r6   r   indexs      r8   prune_headsDeiTAttention.prune_heads  s   u:?7>>55t~~7Y7Y[_[l[l

  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r:   r   r   r   c                 f    U R                  XU5      nU R                  US   U5      nU4USS  -   nU$ )Nr   r   )r   r   )r6   r   r   r   self_outputsattention_outputr   s          r8   rg   DeiTAttention.forward.  sC     ~~m@QR;;|AF#%QR(88r:   )r   r   r   ri   )rj   rk   rl   rm   r   r&   r   rq   r   r(   rp   r   ro   r   r   rg   rs   rt   ru   s   @r8   r   r     s    "z "d ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	F r:   r   c                   n   ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	DeiTIntermediatei=  r    r"   Nc                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r   )r%   r&   r	   r   r*   intermediate_sizer   r}   
hidden_actstrr   intermediate_act_fnr   s     r8   r&   DeiTIntermediate.__init__>  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r:   r   c                 J    U R                  U5      nU R                  U5      nU$ r   r   r   )r6   r   s     r8   rg   DeiTIntermediate.forwardF  s&    

=100?r:   r   rj   rk   rl   rm   r   r&   r(   rp   rg   rs   rt   ru   s   @r8   r   r   =  s6    9z 9d 9U\\ ell  r:   r   c                      ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  S\R                  4S jrS	r	U =r
$ )

DeiTOutputiN  r    r"   Nc                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g r   )
r%   r&   r	   r   r   r*   r   r2   r3   r4   r   s     r8   r&   DeiTOutput.__init__O  sB    YYv779K9KL
zz&"<"<=r:   r   r   c                 R    U R                  U5      nU R                  U5      nX-   nU$ r   r   r   s      r8   rg   DeiTOutput.forwardT  s,    

=1]3%4r:   r   r   ru   s   @r8   r   r   N  sD    >z >d >
U\\  RWR^R^  r:   r   c                      ^  \ rS rSrSrS\SS4U 4S jjr  SS\R                  S\	\R                     S	\
S\\\R                  \R                  4   \\R                     4   4S
 jjrSrU =r$ )	DeiTLayeri^  z?This corresponds to the Block class in the timm implementation.r    r"   Nc                 j  > [         TU ]  5         UR                  U l        SU l        [	        U5      U l        [        U5      U l        [        U5      U l	        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  S9U l        g )Nr   eps)r%   r&   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r	   	LayerNormr*   layer_norm_epslayernorm_beforelayernorm_afterr   s     r8   r&   DeiTLayer.__init__a  s    '-'E'E$&v.,V4 ( "V-?-?VEZEZ [!||F,>,>FDYDYZr:   r   r   r   c                     U R                  U R                  U5      UUS9nUS   nUSS  nXQ-   nU R                  U5      nU R                  U5      nU R	                  Xq5      nU4U-   nU$ )N)r   r   r   )r   r  r  r   r   )r6   r   r   r   self_attention_outputsr   r   layer_outputs           r8   rg   DeiTLayer.forwardk  s     "&!!-0/ "0 "

 2!4(, )8 ++M:((6 {{<?/G+r:   )r   r   r   r  r  r   r   ri   )rj   rk   rl   rm   rn   r   r&   r(   rp   r   ro   r   r   rg   rs   rt   ru   s   @r8   r   r   ^  s    I[z [d [ -1"'	|| ELL)  	
 
uU\\5<</0%2EE	F r:   r   c                      ^  \ rS rSrS\SS4U 4S jjr    SS\R                  S\\R                     S\	S	\	S
\	S\
\\4   4S jjrSrU =r$ )DeiTEncoderi  r    r"   Nc                    > [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf ri   )
r%   r&   r    r	   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r6   r    r_   r7   s      r8   r&   DeiTEncoder.__init__  sR    ]]uVE]E]?^#_?^!If$5?^#_`
&+# $`s   A&r   r   r   output_hidden_statesreturn_dictc                    U(       a  SOS nU(       a  SOS n[        U R                  5       Hz  u  pU(       a  Xa4-   nUb  X(   OS n
U R                  (       a0  U R                  (       a  U R	                  U	R
                  UU
U5      nO	U	" XU5      nUS   nU(       d  Mr  X{S   4-   nM|     U(       a  Xa4-   nU(       d  [        S XU4 5       5      $ [        UUUS9$ )N r   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   r  ).0vs     r8   	<genexpr>&DeiTEncoder.forward.<locals>.<genexpr>  s     m$[q$[s   	)last_hidden_stater   
attentions)	enumerater  r  r   _gradient_checkpointing_func__call__tupler   )r6   r   r   r   r  r  all_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r8   rg   DeiTEncoder.forward  s     #7BD$5b4(4OA#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]M^ _)!,M  &91=M<O&O#'  5*   14D Dm]GZ$[mmm++*
 	
r:   )r    r  r  )NFFT)rj   rk   rl   rm   r   r&   r(   rp   r   ro   r   r  r   rg   rs   rt   ru   s   @r8   r	  r	    s    ,z ,d , -1"'%* )
||)
 ELL))
  	)

 #)
 )
 
uo%	&)
 )
r:   r	  c                       \ rS rSr\rSrSrSrS/r	Sr
SrS\\R                  \R                  \R                   4   SS4S	 jrS
rg)DeiTPreTrainedModeli  deitrX   Tr   r   r"   Nc                 .   [        U[        R                  [        R                  45      (       a  [        R                  R                  UR                  R                  R                  [        R                  5      SU R                  R                  S9R                  UR                  R                  5      UR                  l        UR                  b%  UR                  R                  R                  5         gg[        U[        R                   5      (       aJ  UR                  R                  R                  5         UR                  R                  R#                  S5        g[        U[$        5      (       a  UR&                  R                  R                  5         UR(                  R                  R                  5         UR*                  R                  R                  5         UR,                  b%  UR,                  R                  R                  5         ggg)zInitialize the weightsr   )meanstdNr[   )r}   r	   r   r   inittrunc_normal_weightdatar   r(   r   r    initializer_ranger   r   zero_r   fill_r   r+   r1   r,   r-   )r6   r   s     r8   _init_weights!DeiTPreTrainedModel._init_weights  sh   fryy"))455 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '--KK""$MM$$S)//!!'')&&++113%%**002  ,!!&&,,. -	 0r:   r  )rj   rk   rl   rm   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_2r   r	   r   r   r   r4  rs   r  r:   r8   r(  r(    sW    L$O&*#$N!/E"))RYY*L$M /RV /r:   r(  c                     ^  \ rS rSrSS\S\S\SS4U 4S jjjrS\4S jrS	 r	\
       SS
\\R                     S\\R                     S\\R                     S\\   S\\   S\\   S\S\\\4   4S jj5       rSrU =r$ )	DeiTModeli  r    add_pooling_layerr!   r"   Nc                   > [         TU ]  U5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  UR                  S9U l        U(       a  [        U5      OSU l        U R                  5         g)z
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
use_mask_token (`bool`, *optional*, defaults to `False`):
    Whether to use a mask token for masked image modeling.
)r!   r   N)r%   r&   r    r   r;   r	  encoderr	   r   r*   r   	layernorm
DeiTPoolerpooler	post_init)r6   r    r?  r!   r7   s       r8   r&   DeiTModel.__init__  si     	 (O"6*f&8&8f>S>ST,=j(4 	r:   c                 .    U R                   R                  $ r   )r;   r/   )r6   s    r8   get_input_embeddingsDeiTModel.get_input_embeddings  s    ///r:   c                     UR                  5        H7  u  p#U R                  R                  U   R                  R	                  U5        M9     g)z
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
class PreTrainedModel
N)itemsrA  r  r   r   )r6   heads_to_pruner  r   s       r8   _prune_headsDeiTModel._prune_heads  s<    
 +002LELLu%//;;EB 3r:   rX   rY   r   r   r  r  rV   c                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUc  [	        S5      eU R                  X0R                   R                  5      nU R                  R                  R                  R                  R                  nUR                  U:w  a  UR                  U5      nU R                  XUS9n	U R                  U	UUUUS9n
U
S   nU R                  U5      nU R                  b  U R                  U5      OSnU(       d  Ub  X4OU4nXSS -   $ [!        UUU
R"                  U
R$                  S9$ )z
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
Nz You have to specify pixel_values)rY   rV   )r   r   r  r  r   r   )r  pooler_outputr   r  )r    r   r  use_return_dictr   get_head_maskr  r;   r/   r   r/  r   r   rA  rB  rD  r   r   r  )r6   rX   rY   r   r   r  r  rV   expected_dtypeembedding_outputencoder_outputssequence_outputpooled_outputhead_outputss                 r8   rg   DeiTModel.forward  s|    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ &&y++2O2OP	 99DDKKQQ/'??>:L??Tl + 
 ,,/!5# ' 
 *!,..98<8OO4UY?L?XO;_n^pL!""555)-')77&11	
 	
r:   )r    r;   rA  rB  rD  )TFNNNNNNF)rj   rk   rl   rm   r   ro   r&   r.   rH  rM  r   r   r(   rp   rr   r   r   r   rg   rs   rt   ru   s   @r8   r>  r>    s    z d [_ lp  &0&9 0C  046:,0,0/3&*).;
u||,;
 "%"2"23;
 ELL)	;

 $D>;
 'tn;
 d^;
 #';
 
u00	1;
 ;
r:   r>  c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )rC  i;  r    c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                     U l	        g r   )
r%   r&   r	   r   r*   pooler_output_sizer   r   
pooler_act
activationr   s     r8   r&   DeiTPooler.__init__<  s>    YYv1163L3LM
 !2!23r:   c                 \    US S 2S4   nU R                  U5      nU R                  U5      nU$ )Nr   )r   r_  )r6   r   first_token_tensorrW  s       r8   rg   DeiTPooler.forwardA  s6     +1a40

#566r:   )r_  r   )	rj   rk   rl   rm   r   r&   rg   rs   rt   ru   s   @r8   rC  rC  ;  s    4z 4
 r:   rC  a\  
    DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )custom_introc                      ^  \ rS rSrS\SS4U 4S jjr\       SS\\R                     S\\R                     S\\R                     S	\\   S
\\   S\\   S\S\\\4   4S jj5       rSrU =r$ )DeiTForMaskedImageModelingiJ  r    r"   Nc                 H  > [         TU ]  U5        [        USSS9U l        [        R
                  " [        R                  " UR                  UR                  S-  UR                  -  SS9[        R                  " UR                  5      5      U l        U R                  5         g )NFT)r?  r!   r$   r   )in_channelsout_channelsry   )r%   r&   r>  r)  r	   
Sequentialr   r*   encoder_strider|   PixelShuffledecoderrE  r   s     r8   r&   #DeiTForMaskedImageModeling.__init__W  s     fdS	}}II"..#22A58K8KK
 OOF112
 	r:   rX   rY   r   r   r  r  rV   c           
         Ub  UOU R                   R                  nU R                  UUUUUUUS9nUS   n	U	SS2SS24   n	U	R                  u  pn[	        US-  5      =pU	R                  SSS5      R                  XX5      n	U R                  U	5      nSnUGb  U R                   R                  U R                   R                  -  nUR                  SUU5      nUR                  U R                   R                  S5      R                  U R                   R                  S5      R                  S5      R                  5       n[        R                  R                  XSS	9nUU-  R!                  5       UR!                  5       S
-   -  U R                   R"                  -  nU(       d  U4USS -   nUb  U4U-   $ U$ [%        UUUR&                  UR(                  S9$ )a  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

Examples:
```python
>>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
>>> import torch
>>> from PIL import Image
>>> import requests

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
>>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

>>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
>>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
>>> # create random boolean mask of shape (batch_size, num_patches)
>>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
>>> list(reconstructed_pixel_values.shape)
[1, 3, 224, 224]
```N)rY   r   r   r  r  rV   r   r   r?   r@   r$   none)	reductiongh㈵>)lossreconstructionr   r  )r    rQ  r)  rG   rq   rK   rJ   rm  r{   r5   repeat_interleaver]   r   r	   rL   l1_losssumr|   r   r   r  )r6   rX   rY   r   r   r  r  rV   r   rV  r`   sequence_lengthr|   r<   r=   reconstructed_pixel_valuesmasked_im_lossrB   rc   reconstruction_lossr   s                        r8   rg   "DeiTForMaskedImageModeling.forwardh  s   L &1%<k$++B]B]))+/!5#%=  
 "!* *!QrT'24C4I4I1
\_c122)11!Q:BB:]ck &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7lr"7"s1D8==?488:PTCTUX\XcXcXpXppN02WQR[@F3A3M^%.YSYY(5!//))	
 	
r:   )rm  r)  rZ  )rj   rk   rl   rm   r   r&   r   r   r(   rp   rr   ro   r   r  r   rg   rs   rt   ru   s   @r8   rf  rf  J  s    z d "  046:,0,0/3&*).R
u||,R
 "%"2"23R
 ELL)	R

 $D>R
 'tnR
 d^R
 #'R
 
u//	0R
 R
r:   rf  z
    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                      ^  \ rS rSrS\SS4U 4S jjr\       SS\\R                     S\\R                     S\\R                     S	\\
   S
\\
   S\\
   S\
S\\\4   4S jj5       rSrU =r$ )DeiTForImageClassificationi  r    r"   Nc                 .  > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        U R                  5         g NF)r?  r   )r%   r&   
num_labelsr>  r)  r	   r   r*   Identity
classifierrE  r   s     r8   r&   #DeiTForImageClassification.__init__  ss      ++f>	 OUN_N_bcNc"))F$6$68I8IJikititiv 	r:   rX   r   labelsr   r  r  rV   c           	      t   Ub  UOU R                   R                  nU R                  UUUUUUS9nUS   n	U R                  U	SS2SSS24   5      n
SnUGb  UR	                  U
R
                  5      nU R                   R                  c  U R                  S:X  a  SU R                   l        OoU R                  S:  aN  UR                  [        R                  :X  d  UR                  [        R                  :X  a  SU R                   l        OSU R                   l        U R                   R                  S:X  aI  [        5       nU R                  S:X  a&  U" U
R                  5       UR                  5       5      nOU" X5      nOU R                   R                  S:X  a=  [        5       nU" U
R                  SU R                  5      UR                  S5      5      nO,U R                   R                  S:X  a  [!        5       nU" X5      nU(       d  U
4USS -   nUb  U4U-   $ U$ [#        UU
UR$                  UR&                  S	9$ )
a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, DeiTForImageClassification
>>> import torch
>>> from PIL import Image
>>> import requests

>>> torch.manual_seed(3)  # doctest: +IGNORE_RESULT
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> # note: we are loading a DeiTForImageClassificationWithTeacher from the hub here,
>>> # so the head will be randomly initialized, hence the predictions will be random
>>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
>>> model = DeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224")

>>> inputs = image_processor(images=image, return_tensors="pt")
>>> outputs = model(**inputs)
>>> logits = outputs.logits
>>> # model predicts one of the 1000 ImageNet classes
>>> predicted_class_idx = logits.argmax(-1).item()
>>> print("Predicted class:", model.config.id2label[predicted_class_idx])
Predicted class: Polaroid camera, Polaroid Land camera
```Nr   r   r  r  rV   r   r   
regressionsingle_label_classificationmulti_label_classificationr?   )rr  logitsr   r  )r    rQ  r)  r  r   deviceproblem_typer  r   r(   longrq   r   squeezer   rN   r
   r   r   r  )r6   rX   r   r  r   r  r  rV   r   rV  r  rr  loss_fctr   s                 r8   rg   "DeiTForImageClassification.forward  s   T &1%<k$++B]B]))/!5#%=  
 "!*Aq!9: YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE$!//))	
 	
r:   )r  r)  r  rZ  )rj   rk   rl   rm   r   r&   r   r   r(   rp   ro   r   r  r   rg   rs   rt   ru   s   @r8   r}  r}    s    
z 
d 
  04,0)-,0/3&*).Y
u||,Y
 ELL)Y
 &	Y

 $D>Y
 'tnY
 d^Y
 #'Y
 
u++	,Y
 Y
r:   r}  c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   S	rg)
+DeiTForImageClassificationWithTeacherOutputi.  a  
Output type of [`DeiTForImageClassificationWithTeacher`].

Args:
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the cls_logits and distillation logits.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
        plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
Nr  
cls_logitsdistillation_logitsr   r  r  )rj   rk   rl   rm   rn   r  r   r(   FloatTensor__annotations__r  r  r   r   r  rs   r  r:   r8   r  r  .  s}    , +/FHU&&'..2J**+27;%"3"34;8<M8E%"3"345<59Ju00129r:   r  a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                      ^  \ rS rSrS\SS4U 4S jjr\      SS\\R                     S\\R                     S\\
   S	\\
   S
\\
   S\
S\\\4   4S jj5       rSrU =r$ )%DeiTForImageClassificationWithTeacheriM  r    r"   Nc                   > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l
        U R                  5         g r  )r%   r&   r  r>  r)  r	   r   r*   r  cls_classifierdistillation_classifierrE  r   s     r8   r&   .DeiTForImageClassificationWithTeacher.__init__Y  s      ++f>	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	$
 	r:   rX   r   r   r  r  rV   c           	      L   Ub  UOU R                   R                  nU R                  UUUUUUS9nUS   nU R                  US S 2SS S 24   5      n	U R	                  US S 2SS S 24   5      n
X-   S-  nU(       d  XU
4USS  -   nU$ [        UU	U
UR                  UR                  S9$ )Nr  r   r   r$   )r  r  r  r   r  )r    rQ  r)  r  r  r  r   r  )r6   rX   r   r   r  r  rV   r   rV  r  r  r  r   s                r8   rg   -DeiTForImageClassificationWithTeacher.forwardj  s     &1%<k$++B]B]))/!5#%=  
 "!*((Aq)AB
"::?1aQR7;ST 2a7*=>LFM:! 3!//))
 	
r:   )r  r)  r  r  )NNNNNF)rj   rk   rl   rm   r   r&   r   r   r(   rp   ro   r   r  r  rg   rs   rt   ru   s   @r8   r  r  M  s    z d "  04,0,0/3&*).&
u||,&
 ELL)&
 $D>	&

 'tn&
 d^&
 #'&
 
uAA	B&
 &
r:   r  )r}  r  rf  r>  r(  )r   )>rn   collections.abcr~   dataclassesr   typingr   r   r   r   r   r(   torch.utils.checkpointr	   torch.nnr
   r   r   activationsr   modeling_outputsr   r   r   r   modeling_utilsr   r   pytorch_utilsr   r   utilsr   r   r   r   configuration_deitr   
get_loggerrj   r   Moduler   r.   rp   floatr   r   r   r   r   r   r   r	  r(  r>  rC  rf  r}  r  r  __all__r  r:   r8   <module>r     s;     ! 8 8    A A !  G Q D D * 
		H	%VRYY Vr")) P %II%<<% 
% <<	%
 U\\*% % %>;		 ;~RYY &$BII $Pryy "  '		 'V0
")) 0
f // / /< [
# [
 [
~  	e
!4 e
e
P g
!4 g
g
T :+ : :< 
9
,? 9

9
xr:   