o
    Zhn                  	   @   s<  d Z ddlZddlZddlmZ ddlmZmZm	Z	 ddl
Z
ddlZ
ddl
mZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZmZ ddlmZ eeZeG dd deZeG dd deZ eG dd deZ!dd Z"dd Z#G dd dej$Z%G dd dej$Z&G dd dej$Z'd@de
j(d e)d!e*d"e
j(fd#d$Z+G d%d& d&ej$Z,G d'd( d(ej$Z-G d)d* d*ej$Z.G d+d, d,ej$Z/G d-d. d.ej$Z0G d/d0 d0ej$Z1G d1d2 d2ej$Z2G d3d4 d4ej$Z3G d5d6 d6ej$Z4eG d7d8 d8eZ5eG d9d: d:e5Z6ed;d<G d=d> d>e5Z7g d?Z8dS )AzPyTorch Donut Swin Transformer model.

This implementation is identical to a regular Swin Transformer, without final layer norm on top of the final hidden
states.    N)	dataclass)OptionalTupleUnion)nn   )ACT2FN)PreTrainedModel) find_pruneable_heads_and_indicesmeshgridprune_linear_layer)ModelOutputauto_docstringlogging	torch_int   )DonutSwinConfigc                   @   sr   e Zd ZU dZdZeej ed< dZ	ee
ejdf  ed< dZee
ejdf  ed< dZee
ejdf  ed< dS )DonutSwinEncoderOutputa  
    DonutSwin encoder's outputs, with potential hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   r   r   r    r   r   \/var/www/auris/lib/python3.10/site-packages/transformers/models/donut/modeling_donut_swin.pyr   '   s   
 r   c                   @      e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eeejdf  ed< dZeeejdf  ed< dZeeejdf  ed< dS )	DonutSwinModelOutputaY  
    DonutSwin model's outputs that also contains a pooling of the last hidden states.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
            Average pooling of the last layer hidden-state.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr   pooler_output.r   r   r   )r   r   r   r   r   r   r   r   r   r#   r   r   r   r   r   r   r   r    r"   I      
 r"   c                   @   r!   )	DonutSwinImageClassifierOutputa  
    DonutSwin outputs for image classification.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nlosslogits.r   r   r   )r   r   r   r   r&   r   r   r   r   r'   r   r   r   r   r   r   r   r    r%   n   r$   r%   c                 C   sR   | j \}}}}| ||| ||| ||} | dddddd d|||}|S )z2
    Partitions the given input into windows.
    r   r   r            shapeviewpermute
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowsr   r   r    window_partition   s   $r8   c                 C   sN   | j d }| d|| || |||} | dddddd d|||} | S )z?
    Merges windows to produce higher resolution features.
    r+   r   r   r   r(   r)   r*   r,   )r7   r2   r4   r5   r6   r   r   r    window_reverse   s   
$r9   c                
       sr   e Zd ZdZd fdd	Zdejdededejfd	d
Z		dde	ej
 de	ej dedeej fddZ  ZS )DonutSwinEmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    Fc                    s   t    t|| _| jj}| jj| _|r tt	
dd|jnd | _|jr5tt	
d|d |j| _nd | _t|j| _t|j| _|j| _|| _d S )Nr   )super__init__DonutSwinPatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr   zeros	embed_dim
mask_tokenZuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)selfrM   use_mask_tokenr?   	__class__r   r    r<      s   


 
zDonutSwinEmbeddings.__init__
embeddingsr4   r5   returnc                 C   s   |j d d }| jj d d }tj s||kr||kr| jS | jddddf }| jddddf }|j d }|| j }	|| j }
t|d }|d|||}|dddd}t	j
j||	|
fdd	d
}|dddddd|}tj||fddS )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr+   g      ?r   r   r(   ZbicubicF)sizemodeZalign_cornersdim)r-   rF   r   jit
is_tracingrL   r   Zreshaper/   r   
functionalZinterpolater.   cat)rN   rR   r4   r5   r?   Znum_positionsZclass_pos_embedZpatch_pos_embedrW   Z
new_heightZ	new_widthZsqrt_num_positionsr   r   r    interpolate_pos_encoding   s(   



z,DonutSwinEmbeddings.interpolate_pos_encodingNpixel_valuesbool_masked_posr\   c                 C   s   |j \}}}}| |\}}	| |}| \}
}}|d ur8| j|
|d}|d|}|d|  ||  }| jd urN|rI|| 	||| }n|| j }| 
|}||	fS )Nr+         ?)r-   r>   rH   rT   rE   expand	unsqueezeZtype_asrF   r\   rK   )rN   r]   r^   r\   _r6   r4   r5   rR   output_dimensionsr3   Zseq_lenZmask_tokensmaskr   r   r    forward   s   



zDonutSwinEmbeddings.forward)F)NF)r   r   r   r   r<   r   Tensorintr\   r   r   
BoolTensorboolr   re   __classcell__r   r   rP   r    r:      s    +r:   c                       sN   e Zd ZdZ fddZdd Zdeej de	ej
e	e f fdd	Z  ZS )
r=   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    s   t    |j|j}}|j|j}}t|tjj	r|n||f}t|tjj	r)|n||f}|d |d  |d |d   }|| _|| _|| _|| _
|d |d  |d |d  f| _tj||||d| _d S )Nr   r   )Zkernel_sizeZstride)r;   r<   
image_sizerL   r6   rD   
isinstancecollectionsabcIterabler?   r@   r   Conv2d
projection)rN   rM   rk   rL   r6   hidden_sizer?   rP   r   r    r<     s   
 "z!DonutSwinPatchEmbeddings.__init__c                 C   s   || j d  dkrd| j d || j d   f}tj||}|| j d  dkr>ddd| j d || j d   f}tj||}|S )Nr   r   )rL   r   rZ   pad)rN   r]   r4   r5   
pad_valuesr   r   r    	maybe_pad  s    z"DonutSwinPatchEmbeddings.maybe_padr]   rS   c                 C   sV   |j \}}}}| |||}| |}|j \}}}}||f}|ddd}||fS )Nr(   r   )r-   ru   rq   flatten	transpose)rN   r]   rb   r6   r4   r5   rR   rc   r   r   r    re   (  s   
z DonutSwinPatchEmbeddings.forward)r   r   r   r   r<   ru   r   r   r   r   rf   rg   re   rj   r   r   rP   r    r=   	  s
    .	r=   c                	       sh   e Zd ZdZejfdee dedejddf fddZ	d	d
 Z
dejdeeef dejfddZ  ZS )DonutSwinPatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`Tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutionrW   
norm_layerrS   Nc                    sB   t    || _|| _tjd| d| dd| _|d| | _d S )Nr)   r(   Fbias)r;   r<   ry   rW   r   Linear	reductionrH   )rN   ry   rW   rz   rP   r   r    r<   B  s
   
zDonutSwinPatchMerging.__init__c                 C   sF   |d dkp|d dk}|r!ddd|d d|d f}t j||}|S )Nr(   r   r   )r   rZ   rs   )rN   r1   r4   r5   Z
should_padrt   r   r   r    ru   I  s
   zDonutSwinPatchMerging.maybe_padr1   input_dimensionsc                 C   s   |\}}|j \}}}|||||}| |||}|d d dd ddd dd d f }|d d dd ddd dd d f }	|d d dd ddd dd d f }
|d d dd ddd dd d f }t||	|
|gd}||dd| }| |}| |}|S )Nr   r(   r   r+   r)   )r-   r.   ru   r   r[   rH   r~   )rN   r1   r   r4   r5   r3   rW   r6   Zinput_feature_0Zinput_feature_1Zinput_feature_2Zinput_feature_3r   r   r    re   Q  s   $$$$

zDonutSwinPatchMerging.forward)r   r   r   r   r   rG   r   rg   Moduler<   ru   r   rf   re   rj   r   r   rP   r    rx   5  s
    **rx           Finput	drop_probtrainingrS   c                 C   sd   |dks|s| S d| }| j d fd| jd   }|tj|| j| jd }|  | || }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    r   r   r   )r   dtypedevice)r-   ndimr   Zrandr   r   Zfloor_div)r   r   r   Z	keep_probr-   Zrandom_tensoroutputr   r   r    	drop_pathl  s   
r   c                       sT   e Zd ZdZddee ddf fddZdejdejfdd	Z	de
fd
dZ  ZS )DonutSwinDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   rS   c                    s   t    || _d S N)r;   r<   r   )rN   r   rP   r   r    r<     s   

zDonutSwinDropPath.__init__r   c                 C   s   t || j| jS r   )r   r   r   rN   r   r   r   r    re     s   zDonutSwinDropPath.forwardc                 C   s   d | jS )Nzp={})formatr   rN   r   r   r    
extra_repr  s   zDonutSwinDropPath.extra_reprr   )r   r   r   r   r   floatr<   r   rf   re   strr   rj   r   r   rP   r    r     s
    r   c                       b   e Zd Z fddZdd Z			ddejdeej d	eej d
ee	 de
ej f
ddZ  ZS )DonutSwinSelfAttentionc                    s
  t    || dkrtd| d| d|| _t|| | _| j| j | _t|tj	j
r0|n||f| _ttd| jd  d d| jd  d  || _t| jd }t| jd }tt||gdd}t|d}|d d d d d f |d d d d d f  }	|	ddd }	|	d d d d df  | jd d 7  < |	d d d d df  | jd d 7  < |	d d d d df  d| jd  d 9  < |	d	}
| d
|
 tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _t|j| _ d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r(   r   Zij)Zindexingr+   relative_position_indexr{   )!r;   r<   
ValueErrornum_attention_headsrg   attention_head_sizeall_head_sizerl   rm   rn   ro   r2   r   rB   r   rC   relative_position_bias_tableZarangestackr   rv   r/   r0   sumZregister_bufferr}   Zqkv_biasquerykeyvaluerI   attention_probs_dropout_probrK   )rN   rM   rW   	num_headsr2   Zcoords_hZcoords_wZcoordsZcoords_flattenZrelative_coordsr   rP   r   r    r<     s8   
*,((,
zDonutSwinSelfAttention.__init__c                 C   s6   |  d d | j| jf }||}|ddddS )Nr+   r   r(   r   r   )rT   r   r   r.   r/   )rN   xZnew_x_shaper   r   r    transpose_for_scores  s   
z+DonutSwinSelfAttention.transpose_for_scoresNFr   attention_mask	head_maskoutput_attentionsrS   c                 C   s  |j \}}}| |}| | |}	| | |}
| |}t||	dd}|t	| j
 }| j| jd }|| jd | jd  | jd | jd  d}|ddd }||d }|d ur|j d }||| || j||}||dd }|d| j||}tjj|dd}| |}|d ur|| }t||
}|dddd }| d d | jf }||}|r||f}|S |f}|S )Nr+   r   r   r(   rV   r   )r-   r   r   r   r   r   matmulrw   mathsqrtr   r   r   r.   r2   r/   r0   ra   r   r   rZ   ZsoftmaxrK   rT   r   )rN   r   r   r   r   r3   rW   r6   Zmixed_query_layerZ	key_layerZvalue_layerZquery_layerZattention_scoresZrelative_position_biasZ
mask_shapeZattention_probsZcontext_layerZnew_context_layer_shapeoutputsr   r   r    re     s@   

&


zDonutSwinSelfAttention.forwardNNF)r   r   r   r<   r   r   rf   r   r   ri   r   re   rj   r   r   rP   r    r     s"    %r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )DonutSwinSelfOutputc                    s*   t    t||| _t|j| _d S r   )r;   r<   r   r}   denserI   r   rK   rN   rM   rW   rP   r   r    r<     s   
zDonutSwinSelfOutput.__init__r   input_tensorrS   c                 C      |  |}| |}|S r   r   rK   )rN   r   r   r   r   r    re     s   

zDonutSwinSelfOutput.forwardr   r   r   r<   r   rf   re   rj   r   r   rP   r    r     s    $r   c                       r   )DonutSwinAttentionc                    s2   t    t||||| _t||| _t | _d S r   )r;   r<   r   rN   r   r   setpruned_heads)rN   rM   rW   r   r2   rP   r   r    r<     s   
zDonutSwinAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   rV   )lenr
   rN   r   r   r   r   r   r   r   r   r   r   union)rN   headsindexr   r   r    prune_heads
  s   zDonutSwinAttention.prune_headsNFr   r   r   r   rS   c                 C   s6   |  ||||}| |d |}|f|dd   }|S )Nr   r   )rN   r   )rN   r   r   r   r   Zself_outputsattention_outputr   r   r   r    re     s   zDonutSwinAttention.forwardr   )r   r   r   r<   r   r   rf   r   r   ri   r   re   rj   r   r   rP   r    r     s"    r   c                       2   e Zd Z fddZdejdejfddZ  ZS )DonutSwinIntermediatec                    sJ   t    t|t|j| | _t|jt	rt
|j | _d S |j| _d S r   )r;   r<   r   r}   rg   	mlp_ratior   rl   Z
hidden_actr   r   intermediate_act_fnr   rP   r   r    r<   +  s
   
zDonutSwinIntermediate.__init__r   rS   c                 C   r   r   )r   r   r   r   r   r    re   3     

zDonutSwinIntermediate.forwardr   r   r   rP   r    r   *  s    r   c                       r   )DonutSwinOutputc                    s4   t    tt|j| || _t|j| _	d S r   )
r;   r<   r   r}   rg   r   r   rI   rJ   rK   r   rP   r   r    r<   ;  s   
zDonutSwinOutput.__init__r   rS   c                 C   r   r   r   r   r   r   r    re   @  r   zDonutSwinOutput.forwardr   r   r   rP   r    r   :  s    r   c                       s   e Zd Zd fdd	Zdd Zdd Zd	d
 Z			ddejde	e
e
f deej dee dee de	ejejf fddZ  ZS )DonutSwinLayerr   r   c                    s   t    |j| _|| _|j| _|| _tj||jd| _	t
|||| jd| _|dkr.t|nt | _tj||jd| _t||| _t||| _d S )N)Zeps)r2   r   )r;   r<   Zchunk_size_feed_forward
shift_sizer2   ry   r   rG   Zlayer_norm_epslayernorm_beforer   	attentionr   Identityr   layernorm_afterr   intermediater   r   )rN   rM   rW   ry   r   drop_path_rater   rP   r   r    r<   H  s   
zDonutSwinLayer.__init__c                 C   sD   t || jkr td| _tj rt t|nt || _d S d S Nr   )minr2   r   r   r   rX   rY   Ztensor)rN   ry   r   r   r    set_shift_and_window_sizeU  s
   
 z(DonutSwinLayer.set_shift_and_window_sizec              	   C   s  | j dkrtjd||df||d}td| j t| j | j  t| j  d f}td| j t| j | j  t| j  d f}d}|D ]}	|D ]}
||d d |	|
d d f< |d7 }qEqAt|| j}|d| j| j }|d|d }||dkt	d|dkt	d}|S d }|S )Nr   r   r   r+   r(   g      Yr   )
r   r   rC   slicer2   r8   r.   ra   Zmasked_fillr   )rN   r4   r5   r   r   Zimg_maskZheight_slicesZwidth_slicescountZheight_sliceZwidth_sliceZmask_windows	attn_maskr   r   r    get_attn_mask]  s.   

$zDonutSwinLayer.get_attn_maskc                 C   sR   | j || j   | j  }| j || j   | j  }ddd|d|f}tj||}||fS r   )r2   r   rZ   rs   )rN   r   r4   r5   	pad_rightZ
pad_bottomrt   r   r   r    ru   y  s
   zDonutSwinLayer.maybe_padNFr   r   r   r   always_partitionrS   c                 C   s  |s|  | n	 |\}}| \}}	}
|}| |}|||||
}| |||\}}|j\}	}}}	| jdkrGtj|| j | j fdd}n|}t	|| j
}|d| j
| j
 |
}| j|||j|jd}| j||||d}|d }|d| j
| j
|
}t|| j
||}| jdkrtj|| j| jfdd}n|}|d dkp|d dk}|r|d d d |d |d d f  }|||| |
}|| | }| |}| |}|| | }|r||d	 f}|S |f}|S )
Nr   )r   r(   )Zshiftsdimsr+   r   )r   r   r*   r   )r   rT   r   r.   ru   r-   r   r   Zrollr8   r2   r   r   r   r   r9   r0   r   r   r   r   )rN   r   r   r   r   r   r4   r5   r3   rb   ZchannelsZshortcutrt   Z
height_padZ	width_padZshifted_hidden_statesZhidden_states_windowsr   Zattention_outputsr   Zattention_windowsZshifted_windowsZ
was_paddedZlayer_outputlayer_outputsr   r   r    re     sN   


$

zDonutSwinLayer.forward)r   r   NFF)r   r   r   r<   r   r   ru   r   rf   r   rg   r   r   ri   re   rj   r   r   rP   r    r   G  s*    
r   c                       sd   e Zd Z fddZ			ddejdeeef deej	 dee
 d	ee
 d
eej fddZ  ZS )DonutSwinStagec                    sh   t     | _| _t fddt|D | _|d ur,|tjd| _	nd | _	d| _
d S )Nc              
      s:   g | ]}t  | |d  dkrdn jd  dqS )r(   r   )rM   rW   ry   r   r   r   )r   r2   ).0irM   rW   r   ry   r   r   r    
<listcomp>  s    	z+DonutSwinStage.__init__.<locals>.<listcomp>)rW   rz   F)r;   r<   rM   rW   r   
ModuleListrangeblocksrG   
downsampleZpointing)rN   rM   rW   ry   depthr   r   r   rP   r   r    r<     s   
	
zDonutSwinStage.__init__NFr   r   r   r   r   rS   c                 C   s   |\}}t | jD ]\}}	|d ur|| nd }
|	|||
||}|d }q	|}| jd urE|d d |d d }}||||f}| ||}n||||f}|||f}|rZ||dd  7 }|S )Nr   r   r(   )	enumerater   r   )rN   r   r   r   r   r   r4   r5   r   layer_modulelayer_head_maskr   !hidden_states_before_downsamplingZheight_downsampledZwidth_downsampledrc   Zstage_outputsr   r   r    re     s"   



zDonutSwinStage.forwardr   )r   r   r   r<   r   rf   r   rg   r   r   ri   re   rj   r   r   rP   r    r     s$    
r   c                       s   e Zd Z fddZ						ddejdeeef deej	 d	ee
 d
ee
 dee
 dee
 dee
 deeef fddZ  ZS )DonutSwinEncoderc                    sp   t    t j_ _dd tjd jt	 jddD t
 fddtjD _d_d S )Nc                 S   s   g | ]}|  qS r   )item)r   r   r   r   r    r     s    z-DonutSwinEncoder.__init__.<locals>.<listcomp>r   cpu)r   c                    s   g | ]E}t  t jd |  d d |  d d |  f j|  j| t jd| t jd|d   |jd k rCtnddqS )r(   r   r   N)rM   rW   ry   r   r   r   r   )r   rg   rD   depthsr   r   
num_layersrx   )r   Zi_layerrM   Zdprr@   rN   r   r    r   	  s    
*F)r;   r<   r   r   r   rM   r   Zlinspacer   r   r   r   r   layersgradient_checkpointing)rN   rM   r@   rP   r   r    r<     s   
$

zDonutSwinEncoder.__init__NFTr   r   r   r   output_hidden_states(output_hidden_states_before_downsamplingr   return_dictrS   c	              	   C   s  |rdnd }	|r
dnd }
|rdnd }|r7|j \}}}|j|g||R  }|dddd}|	|f7 }	|
|f7 }
t| jD ]\}}|d urH|| nd }| jr\| jr\| |j|||||}n||||||}|d }|d }|d }|d |d f}|r|r|j \}}}|j|g|d |d f|R  }|dddd}|	|f7 }	|
|f7 }
n'|r|s|j \}}}|j|g||R  }|dddd}|	|f7 }	|
|f7 }
|r||dd  7 }q<|st	dd	 ||	|fD S t
||	||
d
S )Nr   r   r   r   r(   r   r+   c                 s   s    | ]	}|d ur|V  qd S r   r   )r   vr   r   r    	<genexpr>]  s    z+DonutSwinEncoder.forward.<locals>.<genexpr>)r   r   r   r   )r-   r.   r/   r   r   r   r   Z_gradient_checkpointing_func__call__tupler   )rN   r   r   r   r   r   r   r   r   Zall_hidden_statesZall_reshaped_hidden_statesZall_self_attentionsr3   rb   rr   Zreshaped_hidden_stater   r   r   r   r   rc   r   r   r    re     sp   

	



zDonutSwinEncoder.forward)NFFFFT)r   r   r   r<   r   rf   r   rg   r   r   ri   r   r   re   rj   r   r   rP   r    r     s6    
	

r   c                   @   s*   e Zd ZeZdZdZdZdgZdd Z	dS )DonutSwinPreTrainedModeldonutr]   Tr   c                 C   s   t |tjtjfr#|jjjd| jjd |j	dur!|j	j
  dS dS t |tjr8|j	j
  |jjd dS t |trW|jdurH|jj
  |jdurU|jj
  dS dS t |trd|jj
  dS dS )zInitialize the weightsr   )meanZstdNr_   )rl   r   r}   rp   weightdataZnormal_rM   Zinitializer_ranger|   Zzero_rG   Zfill_r:   rE   rF   r   r   )rN   moduler   r   r    _init_weightsp  s"   




z&DonutSwinPreTrainedModel._init_weightsN)
r   r   r   r   Zconfig_classZbase_model_prefixZmain_input_nameZsupports_gradient_checkpointingZ_no_split_modulesr   r   r   r   r    r   g  s    r   c                       s   e Zd Zd fdd	Zdd Zdd Ze													dd
eej	 deej
 deej	 dee dee dedee deeef fddZ  ZS )DonutSwinModelTFc                    sv   t  | || _t|j| _t|jd| jd   | _t	||d| _
t|| j
j| _|r2tdnd| _|   dS )z
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        r(   r   )rO   N)r;   r<   rM   r   r   r   rg   rD   num_featuresr:   rR   r   rA   encoderr   ZAdaptiveAvgPool1dpooler	post_init)rN   rM   Zadd_pooling_layerrO   rP   r   r    r<     s   zDonutSwinModel.__init__c                 C   s   | j jS r   )rR   r>   r   r   r   r    get_input_embeddings  s   z#DonutSwinModel.get_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr   layerr   r   )rN   Zheads_to_pruner  r   r   r   r    _prune_heads  s   zDonutSwinModel._prune_headsNr]   r^   r   r   r   r\   r   rS   c                 C   s   |dur|n| j j}|dur|n| j j}|dur|n| j j}|du r&td| |t| j j}| j|||d\}}	| j	||	||||d}
|
d }d}| j
dur_| 
|dd}t|d}|sm||f|
dd  }|S t|||
j|
j|
jdS )	z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)r^   r\   )r   r   r   r   r   r   r(   )r   r#   r   r   r   )rM   r   r   use_return_dictr   Zget_head_maskr   r   rR   r   r   rw   r   rv   r"   r   r   r   )rN   r]   r^   r   r   r   r\   r   Zembedding_outputr   Zencoder_outputsZsequence_outputpooled_outputr   r   r   r    re     sB   
	
zDonutSwinModel.forward)TFNNNNNFN)r   r   r   r<   r  r  r   r   r   r   rh   ri   r   r   r"   re   rj   r   r   rP   r    r     s:    
	r   a  
    DonutSwin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune DonutSwin on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )Zcustom_introc                       s   e Zd Z fddZe							ddeej deej deej dee	 d	ee	 d
e	dee	 de
eef fddZ  ZS )DonutSwinForImageClassificationc                    sP   t  | |j| _t|| _|jdkrt| jj|jnt | _	| 
  d S r   )r;   r<   Z
num_labelsr   r   r   r}   r   r   
classifierr  )rN   rM   rP   r   r    r<     s   
"z(DonutSwinForImageClassification.__init__NFr]   r   labelsr   r   r\   r   rS   c                 C   s   |dur|n| j j}| j||||||d}|d }	| |	}
d}|dur.| j|
||
| j d}|sD|
f|dd  }|durB|f| S |S t||
|j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   r   r\   r   r   )r'   r  Zpooled_logitsrM   r(   )r&   r'   r   r   r   )	rM   r  r   r
  Zloss_functionr%   r   r   r   )rN   r]   r   r  r   r   r\   r   r   r  r'   r&   r   r   r   r    re     s0   	
z'DonutSwinForImageClassification.forwardr  )r   r   r   r<   r   r   r   r   Z
LongTensorri   r   r   r%   re   rj   r   r   rP   r    r	    s6    
	r	  )r   r   r	  )r   F)9r   collections.abcrm   r   dataclassesr   typingr   r   r   r   Ztorch.utils.checkpointr   Zactivationsr   Zmodeling_utilsr	   Zpytorch_utilsr
   r   r   utilsr   r   r   r   Zconfiguration_donut_swinr   Z
get_loggerr   loggerr   r"   r%   r8   r9   r   r:   r=   rx   rf   r   ri   r   r   r   r   r   r   r   r   r   r   r   r   r	  __all__r   r   r   r    <module>   sX   
 #$], 7e'~=ea@