
    fThn                        S SK rS SKJrJrJrJrJrJrJ	r	  S SK
r
S SKJr  S SKJrJrJr  SSKJr  SSKJrJrJr  SSKJrJr  SSKJrJr  SS	KJrJrJr  S
SK J!r!  \RD                  " \#5      r$ " S S\RJ                  5      r& " S S\RJ                  5      r'\ " S S\5      5       r( S1S\RJ                  S\
RR                  S\
RR                  S\
RR                  S\\
RR                     S\*S\*4S jjr+ " S S\RJ                  5      r, " S S\RJ                  5      r- " S S\RJ                  5      r. " S  S!\RJ                  5      r/ " S" S#\RJ                  5      r0 " S$ S%\RJ                  5      r1 " S& S'\RJ                  5      r2 " S( S)\RJ                  5      r3\ " S* S+\(5      5       r4\" S,S-9 " S. S/\(5      5       r5/ S0Qr6g)2    N)CallableDictListOptionalSetTupleUnion)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging	torch_int   )IJepaConfigc                   n   ^  \ rS rSrSrU 4S jrS	S\R                  S\S\R                  4S jjr	Sr
U =r$ )
IJepaPatchEmbeddings   z
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                   > [         TU ]  5         UR                  UR                  p2UR                  UR
                  pT[        U[        R                  R                  5      (       a  UOX"4n[        U[        R                  R                  5      (       a  UOX34nUS   US   -  US   US   -  -  nX l        X0l        X@l        X`l
        [        R                  " XEX3S9U l        g )Nr   r   )kernel_sizestride)super__init__
image_size
patch_sizenum_channelshidden_size
isinstancecollectionsabcIterablenum_patchesnnConv2d
projection)selfconfigr#   r$   r%   r&   r+   	__class__s          `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/ijepa/modeling_ijepa.pyr"   IJepaPatchEmbeddings.__init__    s    !'!2!2F4E4EJ$*$7$79K9Kk#-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
15*Q-:VW=:XY$$(&))L:i    pixel_valuesinterpolate_pos_encodingreturnc                    UR                   u  p4pVX@R                  :w  a  [        SU R                   SU S35      eU(       dV  XPR                  S   :w  d  X`R                  S   :w  a2  [        SU SU SU R                  S    SU R                  S    S	3	5      eU R	                  U5      R                  S
5      R                  SS
5      nU$ )NzoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .r   r   zInput image size (*z) doesn't match model (z).   )shaper%   
ValueErrorr#   r.   flatten	transpose)r/   r5   r6   
batch_sizer%   heightwidth
embeddingss           r2   forwardIJepaPatchEmbeddings.forward/   s    2>2D2D/
&,,,!../yaI  (++u8J/J (% 9+,Adooa.@-AE  __\2::1=GG1M
r4   )r#   r%   r+   r$   r.   F)__name__
__module____qualname____firstlineno____doc__r"   torchTensorboolrD   __static_attributes____classcell__r1   s   @r2   r   r      s8    jELL D ]b]i]i  r4   r   c            	          ^  \ rS rSrSrSS\S\SS4U 4S jjjrS\R                  S	\
S
\
S\R                  4S jr  SS\R                  S\\R                     S\S\R                  4S jjrSrU =r$ )IJepaEmbeddings@   zZ
Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
r0   use_mask_tokenr7   Nc                   > [         TU ]  5         U(       a6  [        R                  " [        R
                  " SSUR                  5      5      OS U l        [        U5      U l	        U R                  R                  n[        R                  " [        R                  " SX1R                  5      5      U l        [        R                  " UR                  5      U l        UR                   U l        Xl        g )Nr   )r!   r"   r,   	ParameterrL   zerosr&   
mask_tokenr   patch_embeddingsr+   randnposition_embeddingsDropouthidden_dropout_probdropoutr$   r0   )r/   r0   rU   r+   r1   s       r2   r"   IJepaEmbeddings.__init__E   s    Q_",,u{{1a9K9K'LMei 4V <++77#%<<A{L^L^0_#` zz&"<"<= ++r4   rC   rA   rB   c                 ,   UR                   S   nU R                  R                   S   n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  nUR                   S   nX R
                  -  nX0R
                  -  n	[        US-  5      n
UR                  SXU5      nUR                  SSSS5      n[        R                  R                  UX4SSS	9nUR                  SSSS5      R                  SSU5      nU$ )
a  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   g      ?r   r   r;   bicubicF)sizemodealign_corners)r<   r\   rL   jit
is_tracingr$   r   reshapepermuter,   
functionalinterpolateview)r/   rC   rA   rB   r+   num_positionspatch_pos_embeddim
new_height	new_widthsqrt_num_positionss              r2   r6   (IJepaEmbeddings.interpolate_pos_encodingO   s    !&&q)0066q9 yy##%%+*F6?+++22r".
__,	&}c'9:)11!5G]`a)11!Q1=--33(	 4 
 *11!Q1=BB1b#Nr4   r5   bool_masked_posr6   c                 n   UR                   u  pEpgU R                  XS9nUbX  UR                   S   n	U R                  R                  XIS5      n
UR	                  S5      R                  U
5      nUSU-
  -  X-  -   nU(       a  XR                  XU5      -   nOXR                  -   nU R                  U5      nU$ )N)r6   r   rb         ?)	r<   rZ   rY   expand	unsqueezetype_asr6   r\   r_   )r/   r5   ru   r6   r@   _rA   rB   rC   
seq_lengthmask_tokensmasks               r2   rD   IJepaEmbeddings.forwardv   s     (4'9'9$
v**<*k
&#))!,J//00LK",,R088ED#sTz2[5GGJ $#&C&CJX]&^^J#&>&>>J\\*-
r4   )r0   r_   rY   rZ   r$   r\   rF   NF)rG   rH   rI   rJ   rK   r   rN   r"   rL   rM   intr6   r   
BoolTensorrD   rO   rP   rQ   s   @r2   rS   rS   @   s    { D T  %5<< % %UX %]b]i]i %T 7;).	ll "%"2"23 #'	
 
 r4   rS   c                       \ rS rSr\rSrSrSrSS/r	Sr
SrS\\R                  \R                  \R                   4   SS	4S
 jrSrg	)IJepaPreTrainedModel   ijepar5   TrS   
IJepaLayermoduler7   Nc                    [        U[        R                  [        R                  45      (       a  [        R                  R                  UR                  R                  R                  [        R                  5      SU R                  R                  S9R                  UR                  R                  5      UR                  l        UR                  b%  UR                  R                  R                  5         gg[        U[        R                   5      (       aJ  UR                  R                  R                  5         UR                  R                  R#                  S5        g[        U[$        5      (       a  [        R                  R                  UR&                  R                  R                  [        R                  5      SU R                  R                  S9R                  UR&                  R                  5      UR&                  l        UR(                  b%  UR(                  R                  R                  5         ggg)zInitialize the weights        )meanstdNrw   )r'   r,   Linearr-   inittrunc_normal_weightdatatorL   float32r0   initializer_rangedtypebiaszero_	LayerNormfill_rS   r\   rY   )r/   r   s     r2   _init_weights"IJepaPreTrainedModel._init_weights   s   fryy"))455 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '--KK""$MM$$S)00.0gg.C.C**//225==AKK11 /D / b++112	 &&+
   ,!!&&,,. - 1r4    )rG   rH   rI   rJ   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_2r	   r,   r   r-   r   r   rO   r   r4   r2   r   r      sZ    L$O&*#*L9N!/E"))RYY*L$M /RV /r4   r   r   querykeyvalueattention_maskscalingr_   c                    [         R                  " XR                  SS5      5      U-  n[        R                  R                  US[         R                  S9R                  UR                  5      n[        R                  R                  XU R                  S9nUb  X-  n[         R                  " X5      n	U	R                  SS5      R                  5       n	X4$ )Nrb   )rp   r   )ptrainingr   r;   )rL   matmulr?   r,   rk   softmaxr   r   r   r_   r   
contiguous)
r   r   r   r   r   r   r_   kwargsattn_weightsattn_outputs
             r2   eager_attention_forwardr      s     <<}}R'<=GL ==((2U]](SVVW\WbWbcL ==((6??([L !#4,,|3K''1-88:K$$r4   c            
          ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jr SS\	\R                     S	\
S\\\R                  \R                  4   \\R                     4   4S
 jjrSrU =r$ )IJepaSelfAttention   r0   r7   Nc                 0  > [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eXl        UR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        UR                  U l        U R                  S-  U l        SU l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        g )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r9   g      F)r   )r!   r"   r&   num_attention_headshasattrr=   r0   r   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr,   r   qkv_biasr   r   r   r/   r0   r1   s     r2   r"   IJepaSelfAttention.__init__   sG    : ::a?PVXhHiHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r4   xc                     UR                  5       S S U R                  U R                  4-   nUR                  U5      nUR	                  SSSS5      $ )Nrb   r   r;   r   r   )rd   r   r   rm   rj   )r/   r   new_x_shapes      r2   transpose_for_scores'IJepaSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r4   	head_maskoutput_attentionsc                    U R                  U R                  U5      5      nU R                  U R                  U5      5      nU R                  U R                  U5      5      n[        nU R
                  R                  S:w  aT  U R
                  R                  S:X  a  U(       a  [        R                  S5        O[        U R
                  R                     nU" U UUUUU R                  U R                  U R                  (       d  SOU R                  S9u  pUR                  5       S S U R                  4-   n
UR!                  U
5      nU(       a  X4nU$ U4nU$ )Neagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r   )r   r   r_   r   )r   r   r   r   r   r0   _attn_implementationloggerwarning_oncer   r   r   r   r   rd   r   ri   )r/   hidden_statesr   r   	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapeoutputss               r2   rD   IJepaSelfAttention.forward   s9    --dhh}.EF	//

=0IJ//

=0IJ(?;;++w6{{//69>O##L
 '>dkk>^>^&_#)<nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EF6G=2 O\M]r4   )
r   r   r0   r   r   r   r   r   r   r   r   )rG   rH   rI   rJ   r   r"   rL   rM   r   r   rN   r	   r   rD   rO   rP   rQ   s   @r2   r   r      s    ]{ ]t ](%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F! !r4   r   c                      ^  \ rS rSrSrS\SS4U 4S jjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )IJepaSelfOutputi  z
The residual connection is defined in IJepaLayer instead of here (as is the case with other models), due to the
layernorm applied before each block.
r0   r7   Nc                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g N)	r!   r"   r,   r   r&   denser]   r^   r_   r   s     r2   r"   IJepaSelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r4   r   input_tensorc                 J    U R                  U5      nU R                  U5      nU$ r   r   r_   r/   r   r   s      r2   rD   IJepaSelfOutput.forward  s$    

=1]3r4   r   )rG   rH   rI   rJ   rK   r   r"   rL   rM   rD   rO   rP   rQ   s   @r2   r   r     sI    
>{ >t >
U\\  RWR^R^  r4   r   c                      ^  \ rS rSrS\SS4U 4S jjrS\\   SS4S jr  SS\	R                  S	\\	R                     S
\S\\\	R                  \	R                  4   \\	R                     4   4S jjrSrU =r$ )IJepaAttentioni   r0   r7   Nc                    > [         TU ]  5         [        U5      U l        [	        U5      U l        [        5       U l        g r   )r!   r"   r   	attentionr   outputsetpruned_headsr   s     r2   r"   IJepaAttention.__init__!  s0    +F3%f-Er4   headsc                 6   [        U5      S:X  a  g [        XR                  R                  U R                  R                  U R
                  5      u  p[        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l	        [        U R                  R                  USS9U R                  l        U R                  R                  [        U5      -
  U R                  l        U R                  R                  U R                  R                  -  U R                  l        U R
                  R                  U5      U l        g )Nr   r   rp   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r/   r   indexs      r2   prune_headsIJepaAttention.prune_heads'  s   u:?7>>55t~~7Y7Y[_[l[l

  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r4   r   r   r   c                 f    U R                  XU5      nU R                  US   U5      nU4USS  -   nU$ )Nr   r   )r   r   )r/   r   r   r   self_outputsattention_outputr   s          r2   rD   IJepaAttention.forward9  sC     ~~m@QR;;|AF#%QR(88r4   )r   r   r   r   )rG   rH   rI   rJ   r   r"   r   r   r   rL   rM   r   rN   r	   r   rD   rO   rP   rQ   s   @r2   r   r      s    "{ "t ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	F r4   r   c                   n   ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	IJepaIntermediateiG  r0   r7   Nc                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r   )r!   r"   r,   r   r&   intermediate_sizer   r'   
hidden_actstrr   intermediate_act_fnr   s     r2   r"   IJepaIntermediate.__init__H  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r4   r   c                 J    U R                  U5      nU R                  U5      nU$ r   r   r   )r/   r   s     r2   rD   IJepaIntermediate.forwardP  s&    

=100?r4   r  rG   rH   rI   rJ   r   r"   rL   rM   rD   rO   rP   rQ   s   @r2   r   r   G  s6    9{ 9t 9U\\ ell  r4   r   c                      ^  \ rS rSrS\SS4U 4S jjrS\R                  S\R                  S\R                  4S jrS	r	U =r
$ )
IJepaOutputiW  r0   r7   Nc                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g r   )
r!   r"   r,   r   r   r&   r   r]   r^   r_   r   s     r2   r"   IJepaOutput.__init__X  sB    YYv779K9KL
zz&"<"<=r4   r   r   c                 R    U R                  U5      nU R                  U5      nX-   nU$ r   r   r   s      r2   rD   IJepaOutput.forward]  s,    

=1]3%4r4   r   r  rQ   s   @r2   r  r  W  sD    >{ >t >
U\\  RWR^R^  r4   r  c                      ^  \ rS rSrSrS\SS4U 4S jjr  SS\R                  S\	\R                     S	\
S\\\R                  \R                  4   \\R                     4   4S
 jjrSrU =r$ )r   if  z?This corresponds to the Block class in the timm implementation.r0   r7   Nc                 j  > [         TU ]  5         UR                  U l        SU l        [	        U5      U l        [        U5      U l        [        U5      U l	        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  S9U l        g )Nr   eps)r!   r"   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater  r   r,   r   r&   layer_norm_epslayernorm_beforelayernorm_afterr   s     r2   r"   IJepaLayer.__init__i  s    '-'E'E$'/-f5!&) "V-?-?VEZEZ [!||F,>,>FDYDYZr4   r   r   r   c                     U R                  U R                  U5      UUS9nUS   nUSS  nXQ-   nU R                  U5      nU R                  U5      nU R	                  Xq5      nU4U-   nU$ )N)r   r   r   )r   r  r  r  r   )r/   r   r   r   self_attention_outputsr   r   layer_outputs           r2   rD   IJepaLayer.forwards  s     "&!!-0/ "0 "

 2!4(, )8 ++M:((6 {{<?/G+r4   )r   r  r  r  r  r   r  r   )rG   rH   rI   rJ   rK   r   r"   rL   rM   r   rN   r	   r   rD   rO   rP   rQ   s   @r2   r   r   f  s    I[{ [t [ -1"'	|| ELL)  	
 
uU\\5<</0%2EE	F r4   r   c                      ^  \ rS rSrS\SS4U 4S jjr    SS\R                  S\\R                     S\	S	\	S
\	S\
\\4   4S jjrSrU =r$ )IJepaEncoderi  r0   r7   Nc                    > [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf r   )
r!   r"   r0   r,   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r/   r0   r{   r1   s      r2   r"   IJepaEncoder.__init__  sR    ]]fF^F^@_#`@_1Jv$6@_#`a
&+# $as   A&r   r   r   output_hidden_statesreturn_dictc                    U(       a  SOS nU(       a  SOS n[        U R                  5       Hz  u  pU(       a  Xa4-   nUb  X(   OS n
U R                  (       a0  U R                  (       a  U R	                  U	R
                  UU
U5      nO	U	" XU5      nUS   nU(       d  Mr  X{S   4-   nM|     U(       a  Xa4-   nU(       d  [        S XU4 5       5      $ [        UUUS9$ )Nr   r   r   c              3   .   #    U  H  oc  M  Uv   M     g 7fr   r   ).0vs     r2   	<genexpr>'IJepaEncoder.forward.<locals>.<genexpr>  s     m$[q$[s   	)last_hidden_stater   
attentions)	enumerater   r!  r   _gradient_checkpointing_func__call__tupler   )r/   r   r   r   r#  r$  all_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r2   rD   IJepaEncoder.forward  s     #7BD$5b4(4OA#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]M^ _)!,M  &91=M<O&O#'  5*   14D Dm]GZ$[mmm++*
 	
r4   )r0   r!  r   )NFFT)rG   rH   rI   rJ   r   r"   rL   rM   r   rN   r	   r0  r   rD   rO   rP   rQ   s   @r2   r  r    s    ,{ ,t , -1"'%* )
||)
 ELL))
  	)

 #)
 )
 
uo%	&)
 )
r4   r  c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )IJepaPooleri  r0   c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                     U l	        g r   )
r!   r"   r,   r   r&   pooler_output_sizer   r   
pooler_act
activationr   s     r2   r"   IJepaPooler.__init__  s>    YYv1163L3LM
 !2!23r4   c                 \    US S 2S4   nU R                  U5      nU R                  U5      nU$ )Nr   )r   r=  )r/   r   first_token_tensorpooled_outputs       r2   rD   IJepaPooler.forward  s6     +1a40

#566r4   )r=  r   )	rG   rH   rI   rJ   r   r"   rD   rO   rP   rQ   s   @r2   r9  r9    s    4{ 4
 r4   r9  c                   "  ^  \ rS rSrSS\S\S\4U 4S jjjrS\4S jrS\	\
\\
   4   SS	4S
 jr\       SS\\R                      S\\R"                     S\\R                      S\\   S\\   S\\   S\\   S\\\4   4S jj5       rSrU =r$ )
IJepaModeli  r0   add_pooling_layerrU   c                   > [         TU ]  U5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  UR                  S9U l        U(       a  [        U5      OSU l        U R                  5         g)z
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
use_mask_token (`bool`, *optional*, defaults to `False`):
    Whether to use a mask token for masked image modeling.
)rU   r  N)r!   r"   r0   rS   rC   r  encoderr,   r   r&   r  	layernormr9  pooler	post_init)r/   r0   rE  rU   r1   s       r2   r"   IJepaModel.__init__  si     	 )&P#F+f&8&8f>S>ST->k&)D 	r4   r7   c                 .    U R                   R                  $ r   )rC   rZ   )r/   s    r2   get_input_embeddingsIJepaModel.get_input_embeddings  s    ///r4   heads_to_pruneNc                     UR                  5        H7  u  p#U R                  R                  U   R                  R	                  U5        M9     g)z
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
class PreTrainedModel
N)itemsrG  r   r   r   )r/   rO  r   r   s       r2   _prune_headsIJepaModel._prune_heads  s<    
 +002LELLu%//;;EB 3r4   r5   ru   r   r   r#  r6   r$  c                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUc  [	        S5      eU R                  X0R                   R                  5      nU R                  R                  R                  R                  R                  nUR                  U:w  a  UR                  U5      nU R                  XUS9n	U R                  U	UUUUS9n
U
S   nU R                  U5      nU R                  b  U R                  U5      OSnU(       d  Ub  X4OU4nXSS -   $ [!        UUU
R"                  U
R$                  S9$ )z
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
Nz You have to specify pixel_values)ru   r6   )r   r   r#  r$  r   r   )r+  pooler_outputr   r,  )r0   r   r#  use_return_dictr=   get_head_maskr  rC   rZ   r.   r   r   r   rG  rH  rI  r   r   r,  )r/   r5   ru   r   r   r#  r6   r$  expected_dtypeembedding_outputencoder_outputssequence_outputrA  head_outputss                 r2   rD   IJepaModel.forward  s|    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ &&y++2O2OP	 99DDKKQQ/'??>:L??Tl + 
 ,,/!5# ' 
 *!,..98<8OO4UY?L?XO;_n^pL!""555)-')77&11	
 	
r4   )r0   rC   rG  rH  rI  )FFNNNNNNN)rG   rH   rI   rJ   r   rN   r"   r   rM  r   r   r   rR  r   r   rL   rM   r   r	   r   r   rD   rO   rP   rQ   s   @r2   rD  rD    s   { t ]a  $0&: 0C4T#Y+? CD C  046:,0,0/337&*;
u||,;
 "%"2"23;
 ELL)	;

 $D>;
 'tn;
 #+4.;
 d^;
 
u00	1;
 ;
r4   rD  a  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )custom_introc                      ^  \ rS rSrS\SS4U 4S jjr\       SS\\R                     S\\R                     S\\R                     S	\\
   S
\\
   S\\
   S\\
   S\\\4   4S jj5       rSrU =r$ )IJepaForImageClassificationi0  r0   r7   Nc                 .  > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        U R                  5         g )NF)rE  r   )r!   r"   
num_labelsrD  r   r,   r   r&   Identity
classifierrJ  r   s     r2   r"   $IJepaForImageClassification.__init__?  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r4   r5   r   labelsr   r#  r6   r$  c           	      z   Ub  UOU R                   R                  nU R                  UUUUUUS9nUS   n	U R                  U	R	                  SS95      n
SnUGb  UR                  U
R                  5      nU R                   R                  c  U R                  S:X  a  SU R                   l        OoU R                  S:  aN  UR                  [        R                  :X  d  UR                  [        R                  :X  a  SU R                   l        OSU R                   l        U R                   R                  S:X  aI  [        5       nU R                  S:X  a&  U" U
R                  5       UR                  5       5      nOU" X5      nOU R                   R                  S:X  a=  [        5       nU" U
R!                  S	U R                  5      UR!                  S	5      5      nO,U R                   R                  S:X  a  [#        5       nU" X5      nU(       d  U
4USS -   nUb  U4U-   $ U$ [%        UU
UR&                  UR(                  S
9$ )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
N)r   r   r#  r6   r$  r   r   r   
regressionsingle_label_classificationmulti_label_classificationrb   )losslogitsr   r,  )r0   rV  r   re  r   r   deviceproblem_typerc  r   rL   longr   r   squeezer   rm   r
   r   r   r,  )r/   r5   r   rg  r   r#  r6   r$  r   r[  rm  rl  loss_fctr   s                 r2   rD   #IJepaForImageClassification.forwardK  s   " &1%<k$++B]B]**/!5%=#  
 "!*!5!5!!5!<=YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE$!//))	
 	
r4   )re  r   rc  r^  )rG   rH   rI   rJ   r   r"   r   r   rL   rM   rN   r	   r0  r   rD   rO   rP   rQ   s   @r2   ra  ra  0  s    
{ 
t 
  04,0)-,0/337&*A
u||,A
 ELL)A
 &	A

 $D>A
 'tnA
 #+4.A
 d^A
 
u++	,A
 A
r4   ra  )r   rD  ra  )r   )7collections.abcr(   typingr   r   r   r   r   r   r	   rL   torch.nnr,   r
   r   r   activationsr   modeling_outputsr   r   r   modeling_utilsr   r   pytorch_utilsr   r   utilsr   r   r   configuration_ijepar   
get_loggerrG   r   Moduler   rS   r   rM   floatr   r   r   r   r   r  r   r  r9  rD  ra  __all__r   r4   r2   <module>r     s    D D D   A A ! b b F Q 7 7 , 
		H	%$299 $NNbii Nb /? / /N %II%<<% 
% <<	%
 U\\*% % %<; ;|bii $$RYY $N		  ")) ' 'T0
299 0
f"))  Z
% Z
 Z
z O
"6 O
O
d Pr4   