o
    Zh2                  	   @   s  d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlZddlZddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZmZ ddlmZ ddlm Z  e!e"Z#eG dd deZ$eG dd deZ%eG dd deZ&eG dd deZ'dd Z(dd Z)G dd dej*Z+G dd dej*Z,G dd  d ej*Z-dJd#ej.d$e/d%e0d&ej.fd'd(Z1G d)d* d*ej*Z2G d+d, d,ej*Z3G d-d. d.ej*Z4G d/d0 d0ej*Z5G d1d2 d2ej*Z6G d3d4 d4ej*Z7G d5d6 d6ej*Z8G d7d8 d8ej*Z9G d9d: d:ej*Z:eG d;d< d<eZ;eG d=d> d>e;Z<ed?d@G dAdB dBe;Z=edCd@G dDdE dEe;Z>edFd@G dGdH dHe;eZ?g dIZ@dS )KzPyTorch Swin Transformer model.    N)	dataclass)OptionalTupleUnion)nn   )ACT2FN)BackboneOutput)PreTrainedModel) find_pruneable_heads_and_indicesmeshgridprune_linear_layer)ModelOutputauto_docstringlogging	torch_int)BackboneMixin   )
SwinConfigc                   @   sr   e Zd ZU dZdZeej ed< dZ	ee
ejdf  ed< dZee
ejdf  ed< dZee
ejdf  ed< dS )SwinEncoderOutputa  
    Swin encoder's outputs, with potential hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   r   r   r    r!   r!   U/var/www/auris/lib/python3.10/site-packages/transformers/models/swin/modeling_swin.pyr   *   s   
 r   c                   @      e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eeejdf  ed< dZeeejdf  ed< dZeeejdf  ed< dS )	SwinModelOutputaT  
    Swin model's outputs that also contains a pooling of the last hidden states.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
            Average pooling of the last layer hidden-state.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr   pooler_output.r   r   r   )r   r   r   r   r   r   r   r   r    r%   r   r   r   r   r!   r!   r!   r"   r$   K      
 r$   c                   @   s   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eeejdf  ed< dZeeejdf  ed< dZeeejdf  ed< ed	d
 ZdS )SwinMaskedImageModelingOutputa  
    Swin masked image model outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
            Masked image modeling (MLM) loss.
        reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Reconstructed pixel values.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nlossreconstruction.r   r   r   c                 C   s   t dt | jS )Nzlogits attribute is deprecated and will be removed in version 5 of Transformers. Please use the reconstruction attribute to retrieve the final output instead.)warningswarnFutureWarningr)   selfr!   r!   r"   logits   s
   z$SwinMaskedImageModelingOutput.logits)r   r   r   r   r(   r   r   r   r    r)   r   r   r   r   propertyr/   r!   r!   r!   r"   r'   o   s   
 r'   c                   @   r#   )	SwinImageClassifierOutputa  
    Swin outputs for image classification.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr(   r/   .r   r   r   )r   r   r   r   r(   r   r   r   r    r/   r   r   r   r   r!   r!   r!   r"   r1      r&   r1   c                 C   sR   | j \}}}}| ||| ||| ||} | dddddd d|||}|S )z2
    Partitions the given input into windows.
    r   r   r            shapeviewpermute
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowsr!   r!   r"   window_partition   s   $rB   c                 C   sN   | j d }| d|| || |||} | dddddd d|||} | S )z?
    Merges windows to produce higher resolution features.
    r5   r   r   r   r2   r3   r4   r6   )rA   r<   r>   r?   r@   r!   r!   r"   window_reverse   s   
$rC   c                
       sr   e Zd ZdZd fdd	Zdejdededejfd	d
Z		dde	ej
 de	ej dedeej fddZ  ZS )SwinEmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    Fc                    s   t    t|| _| jj}| jj| _|r tt	
dd|jnd | _|jr5tt	
d|d |j| _nd | _t|j| _t|j| _|j| _|| _d S )Nr   )super__init__SwinPatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr   zeros	embed_dim
mask_tokenZuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)r.   rW   use_mask_tokenrI   	__class__r!   r"   rF      s   


 
zSwinEmbeddings.__init__
embeddingsr>   r?   returnc                 C   s   |j d d }| jj d d }tj s||kr||kr| jS | jddddf }| jddddf }|j d }|| j }	|| j }
t|d }|d|||}|dddd}t	j
j||	|
fdd	d
}|dddddd|}tj||fddS )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr5         ?r   r   r2   ZbicubicF)sizemodeZalign_cornersdim)r7   rP   r   jit
is_tracingrV   r   reshaper9   r   
functionalZinterpolater8   cat)r.   r[   r>   r?   rI   Znum_positionsZclass_pos_embedZpatch_pos_embedra   Z
new_heightZ	new_widthZsqrt_num_positionsr!   r!   r"   interpolate_pos_encoding   s(   



z'SwinEmbeddings.interpolate_pos_encodingNpixel_valuesbool_masked_posrg   c                 C   s   |j \}}}}| |\}}	| |}| \}
}}|d ur8| j|
|d}|d|}|d|  ||  }| jd urN|rI|| 	||| }n|| j }| 
|}||	fS )Nr5         ?)r7   rH   rR   r^   rO   expand	unsqueezeZtype_asrP   rg   rU   )r.   rh   ri   rg   _r@   r>   r?   r[   output_dimensionsr=   Zseq_lenZmask_tokensmaskr!   r!   r"   forward  s   



zSwinEmbeddings.forward)F)NF)r   r   r   r   rF   r   Tensorintrg   r   r   
BoolTensorboolr   rp   __classcell__r!   r!   rY   r"   rD      s    +rD   c                       sN   e Zd ZdZ fddZdd Zdeej de	ej
e	e f fdd	Z  ZS )
rG   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    s   t    |j|j}}|j|j}}t|tjj	r|n||f}t|tjj	r)|n||f}|d |d  |d |d   }|| _|| _|| _|| _
|d |d  |d |d  f| _tj||||d| _d S )Nr   r   )kernel_sizeZstride)rE   rF   
image_sizerV   r@   rN   
isinstancecollectionsabcIterablerI   rJ   r   Conv2d
projection)r.   rW   rw   rV   r@   hidden_sizerI   rY   r!   r"   rF   9  s   
 "zSwinPatchEmbeddings.__init__c                 C   s   || j d  dkrd| j d || j d   f}tj||}|| j d  dkr>ddd| j d || j d   f}tj||}|S )Nr   r   )rV   r   re   pad)r.   rh   r>   r?   
pad_valuesr!   r!   r"   	maybe_padH  s    zSwinPatchEmbeddings.maybe_padrh   r\   c                 C   sV   |j \}}}}| |||}| |}|j \}}}}||f}|ddd}||fS )Nr2   r   )r7   r   r}   flatten	transpose)r.   rh   rm   r@   r>   r?   r[   rn   r!   r!   r"   rp   Q  s   
zSwinPatchEmbeddings.forward)r   r   r   r   rF   r   r   r   r   r   rq   rr   rp   ru   r!   r!   rY   r"   rG   2  s
    .	rG   c                	       sh   e Zd ZdZejfdee dedejddf fddZ	d	d
 Z
dejdeeef dejfddZ  ZS )SwinPatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`Tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutionra   
norm_layerr\   Nc                    sB   t    || _|| _tjd| d| dd| _|d| | _d S )Nr3   r2   Fbias)rE   rF   r   ra   r   Linear	reductionrR   )r.   r   ra   r   rY   r!   r"   rF   j  s
   
zSwinPatchMerging.__init__c                 C   sF   |d dkp|d dk}|r!ddd|d d|d f}t j||}|S )Nr2   r   r   )r   re   r   )r.   r;   r>   r?   Z
should_padr   r!   r!   r"   r   q  s
   zSwinPatchMerging.maybe_padr;   input_dimensionsc                 C   s   |\}}|j \}}}|||||}| |||}|d d dd ddd dd d f }|d d dd ddd dd d f }	|d d dd ddd dd d f }
|d d dd ddd dd d f }t||	|
|gd}||dd| }| |}| |}|S )Nr   r2   r   r5   r3   )r7   r8   r   r   rf   rR   r   )r.   r;   r   r>   r?   r=   ra   r@   Zinput_feature_0Zinput_feature_1Zinput_feature_2Zinput_feature_3r!   r!   r"   rp   y  s   $$$$

zSwinPatchMerging.forward)r   r   r   r   r   rQ   r   rr   ModulerF   r   r   rq   rp   ru   r!   r!   rY   r"   r   ]  s
    **r           Finput	drop_probtrainingr\   c                 C   sd   |dks|s| S d| }| j d fd| jd   }|tj|| j| jd }|  | || }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    r   r   r   )r   dtypedevice)r7   ndimr   Zrandr   r   Zfloor_div)r   r   r   Z	keep_probr7   Zrandom_tensoroutputr!   r!   r"   	drop_path  s   
r   c                       sT   e Zd ZdZddee ddf fddZdejdejfdd	Z	de
fd
dZ  ZS )SwinDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   r\   c                    s   t    || _d S N)rE   rF   r   )r.   r   rY   r!   r"   rF     s   

zSwinDropPath.__init__r   c                 C   s   t || j| jS r   )r   r   r   r.   r   r!   r!   r"   rp     s   zSwinDropPath.forwardc                 C   s   d | jS )Nzp={})formatr   r-   r!   r!   r"   
extra_repr  s   zSwinDropPath.extra_reprr   )r   r   r   r   r   floatrF   r   rq   rp   strr   ru   r!   r!   rY   r"   r     s
    r   c                       b   e Zd Z fddZdd Z			ddejdeej d	eej d
ee	 de
ej f
ddZ  ZS )SwinSelfAttentionc                    s
  t    || dkrtd| d| d|| _t|| | _| j| j | _t|tj	j
r0|n||f| _ttd| jd  d d| jd  d  || _t| jd }t| jd }tt||gdd}t|d}|d d d d d f |d d d d d f  }	|	ddd }	|	d d d d df  | jd d 7  < |	d d d d df  | jd d 7  < |	d d d d df  d| jd  d 9  < |	d	}
| d
|
 tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _t|j| _ d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r2   r   Zij)Zindexingr5   relative_position_indexr   )!rE   rF   
ValueErrornum_attention_headsrr   attention_head_sizeall_head_sizerx   ry   rz   r{   r<   r   rL   r   rM   relative_position_bias_tableZarangestackr   r   r9   r:   sumZregister_bufferr   Zqkv_biasquerykeyvaluerS   attention_probs_dropout_probrU   )r.   rW   ra   	num_headsr<   Zcoords_hZcoords_wZcoordsZcoords_flattenZrelative_coordsr   rY   r!   r"   rF     s8   
*,((,
zSwinSelfAttention.__init__c                 C   s6   |  d d | j| jf }||}|ddddS )Nr5   r   r2   r   r   )r^   r   r   r8   r9   )r.   xZnew_x_shaper!   r!   r"   transpose_for_scores  s   
z&SwinSelfAttention.transpose_for_scoresNFr   attention_mask	head_maskoutput_attentionsr\   c                 C   s  |j \}}}| |}| | |}	| | |}
| |}t||	dd}|t	| j
 }| j| jd }|| jd | jd  | jd | jd  d}|ddd }||d }|d ur|j d }||| || j||}||dd }|d| j||}tjj|dd}| |}|d ur|| }t||
}|dddd }| d d | jf }||}|r||f}|S |f}|S )Nr5   r   r   r2   r`   r   )r7   r   r   r   r   r   matmulr   mathsqrtr   r   r   r8   r<   r9   r:   rl   r   r   re   ZsoftmaxrU   r^   r   )r.   r   r   r   r   r=   ra   r@   Zmixed_query_layerZ	key_layerZvalue_layerZquery_layerZattention_scoresZrelative_position_biasZ
mask_shapeZattention_probsZcontext_layerZnew_context_layer_shapeoutputsr!   r!   r"   rp     s@   

&


zSwinSelfAttention.forwardNNF)r   r   r   rF   r   r   rq   r   r   rt   r   rp   ru   r!   r!   rY   r"   r     s"    %r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )SwinSelfOutputc                    s*   t    t||| _t|j| _d S r   )rE   rF   r   r   denserS   r   rU   r.   rW   ra   rY   r!   r"   rF     s   
zSwinSelfOutput.__init__r   input_tensorr\   c                 C      |  |}| |}|S r   r   rU   )r.   r   r   r!   r!   r"   rp   !  s   

zSwinSelfOutput.forwardr   r   r   rF   r   rq   rp   ru   r!   r!   rY   r"   r     s    $r   c                       r   )SwinAttentionc                    s2   t    t||||| _t||| _t | _d S r   )rE   rF   r   r.   r   r   setpruned_heads)r.   rW   ra   r   r<   rY   r!   r"   rF   )  s   
zSwinAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   r`   )lenr   r.   r   r   r   r   r   r   r   r   r   r   union)r.   headsindexr!   r!   r"   prune_heads/  s   zSwinAttention.prune_headsNFr   r   r   r   r\   c                 C   s6   |  ||||}| |d |}|f|dd   }|S )Nr   r   )r.   r   )r.   r   r   r   r   Zself_outputsattention_outputr   r!   r!   r"   rp   A  s   zSwinAttention.forwardr   )r   r   r   rF   r   r   rq   r   r   rt   r   rp   ru   r!   r!   rY   r"   r   (  s"    r   c                       2   e Zd Z fddZdejdejfddZ  ZS )SwinIntermediatec                    sJ   t    t|t|j| | _t|jt	rt
|j | _d S |j| _d S r   )rE   rF   r   r   rr   	mlp_ratior   rx   Z
hidden_actr   r   intermediate_act_fnr   rY   r!   r"   rF   O  s
   
zSwinIntermediate.__init__r   r\   c                 C   r   r   )r   r   r   r!   r!   r"   rp   W     

zSwinIntermediate.forwardr   r!   r!   rY   r"   r   N  s    r   c                       r   )
SwinOutputc                    s4   t    tt|j| || _t|j| _	d S r   )
rE   rF   r   r   rr   r   r   rS   rT   rU   r   rY   r!   r"   rF   ^  s   
zSwinOutput.__init__r   r\   c                 C   r   r   r   r   r!   r!   r"   rp   c  r   zSwinOutput.forwardr   r!   r!   rY   r"   r   ]  s    r   c                       s   e Zd Zd fdd	Zdd Zdd Zd	d
 Z			ddejde	e
e
f deej dee dee de	ejejf fddZ  ZS )	SwinLayerr   r   c                    s   t    |j| _|| _|j| _|| _tj||jd| _	t
|||| jd| _|dkr.t|nt | _tj||jd| _t||| _t||| _d S )NZeps)r<   r   )rE   rF   Zchunk_size_feed_forward
shift_sizer<   r   r   rQ   layer_norm_epslayernorm_beforer   	attentionr   Identityr   layernorm_afterr   intermediater   r   )r.   rW   ra   r   r   drop_path_rater   rY   r!   r"   rF   j  s   
zSwinLayer.__init__c                 C   sD   t || jkr td| _tj rt t|nt || _d S d S Nr   )minr<   r   r   r   rb   rc   Ztensor)r.   r   r!   r!   r"   set_shift_and_window_sizew  s
   
 z#SwinLayer.set_shift_and_window_sizec              	   C   s  | j dkrtjd||df||d}td| j t| j | j  t| j  d f}td| j t| j | j  t| j  d f}d}|D ]}	|D ]}
||d d |	|
d d f< |d7 }qEqAt|| j}|d| j| j }|d|d }||dkt	d|dkt	d}|S d }|S )Nr   r   r   r5   r2   g      Yr   )
r   r   rM   slicer<   rB   r8   rl   Zmasked_fillr   )r.   r>   r?   r   r   Zimg_maskZheight_slicesZwidth_slicescountZheight_sliceZwidth_sliceZmask_windows	attn_maskr!   r!   r"   get_attn_mask  s.   

$zSwinLayer.get_attn_maskc                 C   sR   | j || j   | j  }| j || j   | j  }ddd|d|f}tj||}||fS r   )r<   r   re   r   )r.   r   r>   r?   	pad_rightZ
pad_bottomr   r!   r!   r"   r     s
   zSwinLayer.maybe_padNFr   r   r   r   always_partitionr\   c                 C   s  |s|  | n	 |\}}| \}}	}
|}| |}|||||
}| |||\}}|j\}	}}}	| jdkrGtj|| j | j fdd}n|}t	|| j
}|d| j
| j
 |
}| j|||j|jd}| j||||d}|d }|d| j
| j
|
}t|| j
||}| jdkrtj|| j| jfdd}n|}|d dkp|d dk}|r|d d d |d |d d f  }|||| |
}|| | }| |}| |}|| | }|r||d	 f}|S |f}|S )
Nr   )r   r2   )Zshiftsdimsr5   r   )r   r   r4   r   )r   r^   r   r8   r   r7   r   r   ZrollrB   r<   r   r   r   r   rC   r:   r   r   r   r   )r.   r   r   r   r   r   r>   r?   r=   rm   channelsZshortcutr   Z
height_padZ	width_padZshifted_hidden_statesZhidden_states_windowsr   Zattention_outputsr   Zattention_windowsZshifted_windowsZ
was_paddedZlayer_outputlayer_outputsr!   r!   r"   rp     sN   


$

zSwinLayer.forward)r   r   NFF)r   r   r   rF   r   r   r   r   rq   r   rr   r   r   rt   rp   ru   r!   r!   rY   r"   r   i  s*    
r   c                       sd   e Zd Z fddZ			ddejdeeef deej	 dee
 d	ee
 d
eej fddZ  ZS )	SwinStagec                    sh   t     | _| _t fddt|D | _|d ur,|tjd| _	nd | _	d| _
d S )Nc              
      s:   g | ]}t  | |d  dkrdn jd  dqS )r2   r   )rW   ra   r   r   r   r   )r   r<   .0irW   ra   r   r   r   r!   r"   
<listcomp>  s    	z&SwinStage.__init__.<locals>.<listcomp>)ra   r   F)rE   rF   rW   ra   r   
ModuleListrangeblocksrQ   
downsampleZpointing)r.   rW   ra   r   depthr   r   r   rY   r   r"   rF     s   
	
zSwinStage.__init__NFr   r   r   r   r   r\   c                 C   s   |\}}t | jD ]\}}	|d ur|| nd }
|	|||
||}|d }q	|}| jd urE|d d |d d }}||||f}| ||}n||||f}|||f}|rZ||dd  7 }|S )Nr   r   r2   )	enumerater   r   )r.   r   r   r   r   r   r>   r?   r   layer_modulelayer_head_maskr   !hidden_states_before_downsamplingZheight_downsampledZwidth_downsampledrn   Zstage_outputsr!   r!   r"   rp     s"   



zSwinStage.forwardr   )r   r   r   rF   r   rq   r   rr   r   r   rt   rp   ru   r!   r!   rY   r"   r     s$    
r   c                       s   e Zd Z fddZ						ddejdeeef deej	 d	ee
 d
ee
 dee
 dee
 dee
 deeef fddZ  ZS )SwinEncoderc                    sp   t    t j_ _dd tjd jt	 jddD t
 fddtjD _d_d S )Nc                 S   s   g | ]}|  qS r!   )item)r   r   r!   r!   r"   r   '  s    z(SwinEncoder.__init__.<locals>.<listcomp>r   cpu)r   c                    s   g | ]E}t  t jd |  d d |  d d |  f j|  j| t jd| t jd|d   |jd k rCtnddqS )r2   r   r   N)rW   ra   r   r   r   r   r   )r   rr   rN   depthsr   r   
num_layersr   )r   Zi_layerrW   ZdprrJ   r.   r!   r"   r   )  s    
*F)rE   rF   r   r   r   rW   r   Zlinspacer   r   r   r   r   layersgradient_checkpointing)r.   rW   rJ   rY   r   r"   rF   #  s   
$

zSwinEncoder.__init__NFTr   r   r   r   output_hidden_states(output_hidden_states_before_downsamplingr   return_dictr\   c	              	   C   s  |rdnd }	|r
dnd }
|rdnd }|r7|j \}}}|j|g||R  }|dddd}|	|f7 }	|
|f7 }
t| jD ]\}}|d urH|| nd }| jr\| jr\| |j|||||}n||||||}|d }|d }|d }|d |d f}|r|r|j \}}}|j|g|d |d f|R  }|dddd}|	|f7 }	|
|f7 }
n'|r|s|j \}}}|j|g||R  }|dddd}|	|f7 }	|
|f7 }
|r||dd  7 }q<|st	dd	 ||	|fD S t
||	||
d
S )Nr!   r   r   r   r2   r   r5   c                 s   s    | ]	}|d ur|V  qd S r   r!   )r   vr!   r!   r"   	<genexpr>}  s    z&SwinEncoder.forward.<locals>.<genexpr>)r   r   r   r   )r7   r8   r9   r   r   r   r   Z_gradient_checkpointing_func__call__tupler   )r.   r   r   r   r   r   r   r   r   Zall_hidden_statesZall_reshaped_hidden_statesZall_self_attentionsr=   rm   r~   Zreshaped_hidden_stater   r   r   r   r   rn   r!   r!   r"   rp   9  sp   

	



zSwinEncoder.forward)NFFFFT)r   r   r   rF   r   rq   r   rr   r   r   rt   r   r   rp   ru   r!   r!   rY   r"   r   "  s6    
	

r   c                   @   s*   e Zd ZeZdZdZdZdgZdd Z	dS )SwinPreTrainedModelswinrh   Tr   c                 C   s   t |tjtjfr#|jjjd| jjd |j	dur!|j	j
  dS dS t |tjr8|j	j
  |jjd dS t |trW|jdurH|jj
  |jdurU|jj
  dS dS t |trd|jj
  dS dS )zInitialize the weightsr   )meanZstdNrj   )rx   r   r   r|   weightdataZnormal_rW   Zinitializer_ranger   Zzero_rQ   Zfill_rD   rO   rP   r   r   )r.   moduler!   r!   r"   _init_weights  s"   




z!SwinPreTrainedModel._init_weightsN)
r   r   r   r   Zconfig_classZbase_model_prefixZmain_input_nameZsupports_gradient_checkpointingZ_no_split_modulesr  r!   r!   r!   r"   r    s    r  c                       s   e Zd Zd fdd	Zdd Zdd Ze													dd
eej	 deej
 deej	 dee dee dedee deeef fddZ  ZS )	SwinModelTFc                    s   t  | || _t|j| _t|jd| jd   | _t	||d| _
t|| j
j| _tj| j|jd| _|r<tdnd| _|   dS )a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r2   r   )rX   r   N)rE   rF   rW   r   r   r   rr   rN   num_featuresrD   r[   r   rK   encoderr   rQ   r   	layernormZAdaptiveAvgPool1dpooler	post_init)r.   rW   add_pooling_layerrX   rY   r!   r"   rF     s   zSwinModel.__init__c                 C      | j jS r   r[   rH   r-   r!   r!   r"   get_input_embeddings     zSwinModel.get_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  layerr   r   )r.   Zheads_to_pruner  r   r!   r!   r"   _prune_heads  s   zSwinModel._prune_headsNrh   ri   r   r   r   rg   r   r\   c                 C   s   |dur|n| j j}|dur|n| j j}|dur|n| j j}|du r&td| |t| j j}| j|||d\}}	| j	||	||||d}
|
d }| 
|}d}| jdurd| |dd}t|d}|sr||f|
dd  }|S t|||
j|
j|
jdS )	z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)ri   rg   )r   r   r   r   r   r   r2   )r   r%   r   r   r   )rW   r   r   use_return_dictr   Zget_head_maskr   r   r[   r  r  r  r   r   r   r$   r   r   r   )r.   rh   ri   r   r   r   rg   r   embedding_outputr   Zencoder_outputssequence_outputpooled_outputr   r!   r!   r"   rp     sD   
	

zSwinModel.forward)TFNNNNNFN)r   r   r   rF   r  r  r   r   r   r   rs   rt   r   r   r$   rp   ru   r!   r!   rY   r"   r    s:    
	r  a\  
    Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )Zcustom_introc                       s   e Zd Z fddZe							ddeej deej deej dee	 d	ee	 d
e	dee	 de
eef fddZ  ZS )SwinForMaskedImageModelingc                    sn   t  | t|ddd| _t|jd|jd   }ttj	||j
d |j ddt|j
| _|   d S )NFT)r  rX   r2   r   )Zin_channelsZout_channelsrv   )rE   rF   r  r  rr   rN   r   r   Z
Sequentialr|   Zencoder_strider@   ZPixelShuffledecoderr  )r.   rW   r  rY   r!   r"   rF     s   
z#SwinForMaskedImageModeling.__init__NFrh   ri   r   r   r   rg   r   r\   c              	   C   s>  |dur|n| j j}| j|||||||d}|d }	|	dd}	|	j\}
}}t|d  }}|	|
|||}	| |	}d}|dur}| j j	| j j
 }|d||}|| j j
d| j j
dd }tjj||dd	}||  | d
  | j j }|s|f|dd  }|dur|f| S |S t|||j|j|jdS )a7  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, SwinForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swin-base-simmim-window6-192")
        >>> model = SwinForMaskedImageModeling.from_pretrained("microsoft/swin-base-simmim-window6-192")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```N)ri   r   r   r   rg   r   r   r   r2   r]   r5   none)r   gh㈵>)r(   r)   r   r   r   )rW   r  r  r   r7   r   floorrd   r   rw   rV   Zrepeat_interleaverl   r:   r   re   Zl1_lossr   r@   r'   r   r   r   )r.   rh   ri   r   r   r   rg   r   r   r  r=   r@   Zsequence_lengthr>   r?   Zreconstructed_pixel_valuesZmasked_im_lossr^   ro   Zreconstruction_lossr   r!   r!   r"   rp   $  sJ   &

 z"SwinForMaskedImageModeling.forwardr  )r   r   r   rF   r   r   r   r   rs   rt   r   r   r'   rp   ru   r!   r!   rY   r"   r    s6    
	r  a  
    Swin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune Swin on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                       s   e Zd Z fddZe							ddeej deej deej dee	 d	ee	 d
e	dee	 de
eef fddZ  ZS )SwinForImageClassificationc                    sP   t  | |j| _t|| _|jdkrt| jj|jnt | _	| 
  d S r   )rE   rF   Z
num_labelsr  r  r   r   r  r   
classifierr  )r.   rW   rY   r!   r"   rF     s   
"z#SwinForImageClassification.__init__NFrh   r   labelsr   r   rg   r   r\   c                 C   s   |dur|n| j j}| j||||||d}|d }	| |	}
d}|dur.| j|
||
| j d}|sD|
f|dd  }|durB|f| S |S t||
|j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   r   rg   r   r   )r/   r%  Zpooled_logitsrW   r2   )r(   r/   r   r   r   )	rW   r  r  r$  Zloss_functionr1   r   r   r   )r.   rh   r   r%  r   r   rg   r   r   r  r/   r(   r   r!   r!   r"   rp     s0   	
z"SwinForImageClassification.forwardr  )r   r   r   rF   r   r   r   r   Z
LongTensorrt   r   r   r1   rp   ru   r!   r!   rY   r"   r#  z  s6    
	r#  zM
    Swin backbone, to be used with frameworks like DETR and MaskFormer.
    c                       s^   e Zd Zdef fddZdd Z			ddejdee	 d	ee	 d
ee	 de
f
ddZ  ZS )SwinBackbonerW   c                    s   t    t     jg fddtt jD  | _t | _	t
 | j	j| _i }t| j| jD ]\}}t|||< q5t|| _|   d S )Nc                    s   g | ]}t  jd |  qS )r2   )rr   rN   r   rW   r!   r"   r     s    z)SwinBackbone.__init__.<locals>.<listcomp>)rE   rF   Z_init_backbonerN   r   r   r   r  rD   r[   r   rK   r  zipZ_out_featuresr   r   rQ   Z
ModuleDicthidden_states_normsr  )r.   rW   r)  stager@   rY   r'  r"   rF     s   &
zSwinBackbone.__init__c                 C   r  r   r  r-   r!   r!   r"   r    r  z!SwinBackbone.get_input_embeddingsNrh   r   r   r   r\   c              
   C   s6  |dur|n| j j}|dur|n| j j}|dur|n| j j}| |\}}| j||d|ddddd}|j}d}	t| j|D ]A\}
}|
| j	v r~|j
\}}}}|dddd }|||| |}| j|
 |}|||||}|dddd }|	|f7 }	q=|s|	f}|r||jf7 }|S t|	|r|jnd|jd	S )
aK  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("shi-labs/nat-mini-in1k-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swin-tiny-patch4-window7-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 7, 7]
        ```NT)r   r   r   r   r   r   r!   r   r2   r   r   )feature_mapsr   r   )rW   r  r   r   r[   r  r   r(  Zstage_namesZout_featuresr7   r9   r:   r8   r)  r   r	   r   )r.   rh   r   r   r   r  r   r   r   r+  r*  Zhidden_stater=   r@   r>   r?   r   r!   r!   r"   rp     sJ    

zSwinBackbone.forward)NNN)r   r   r   r   rF   r  r   rq   r   rt   r	   rp   ru   r!   r!   rY   r"   r&    s"    r&  )r#  r  r  r  r&  )r   F)Ar   collections.abcry   r   r*   dataclassesr   typingr   r   r   r   Ztorch.utils.checkpointr   Zactivationsr   Zmodeling_outputsr	   Zmodeling_utilsr
   Zpytorch_utilsr   r   r   utilsr   r   r   r   Zutils.backbone_utilsr   Zconfiguration_swinr   Z
get_loggerr   loggerr   r$   r'   r1   rB   rC   r   rD   rG   r   rq   r   rt   r   r   r   r   r   r   r   r   r   r   r  r  r  r#  r&  __all__r!   r!   r!   r"   <module>   sr   
 #,#
\+ 7d&}<ecg@b