o
    ZhS                     @   s.  d Z ddlZddlmZmZmZ ddlZddlZddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ e rpddl Z ddl!m"Z" ejj#j$e j%j&j'_$e(e)Z*G dd dej#Z+G dd dej#Z,G dd dej#Z-G dd dej#Z.G dd dej#Z/G dd dej#Z0G dd dej#Z1d8d d!Z2G d"d# d#ej#Z3eG d$d% d%eZ4d9d&d'Z5G d(d) d)ej#Z6G d*d+ d+ej#Z7eG d,d- d-e4Z8ed.d/G d0d1 d1e4Z9ed2d/G d3d4 d4e4Z:eG d5d6 d6e4Z;g d7Z<dS ):zPyTorch LayoutLMv2 model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward)auto_docstringis_detectron2_availableloggingrequires_backends   )LayoutLMv2Config)META_ARCH_REGISTRYc                       s(   e Zd ZdZ fddZdd Z  ZS )LayoutLMv2EmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t t|   tj|j|j|jd| _t|j	|j| _
t|j|j| _t|j|j| _t|j|j| _t|j|j| _t|j|j| _tj|j|jd| _t|j| _| jdt|j	ddd d S )N)padding_idxepsposition_ids)r   F
persistent)superr   __init__r   	EmbeddingZ
vocab_sizehidden_sizeZpad_token_idword_embeddingsZmax_position_embeddingsposition_embeddingsZmax_2d_position_embeddingsZcoordinate_sizex_position_embeddingsy_position_embeddingsZ
shape_sizeh_position_embeddingsw_position_embeddingsZtype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig	__class__ a/var/www/auris/lib/python3.10/site-packages/transformers/models/layoutlmv2/modeling_layoutlmv2.pyr"   6   s   
zLayoutLMv2Embeddings.__init__c           
   
   C   s  z:|  |d d d d df }| |d d d d df }|  |d d d d df }| |d d d d df }W n tyK } ztd|d }~ww | |d d d d df |d d d d df  }| |d d d d df |d d d d df  }tj||||||gdd}	|	S )Nr   r      r	   z;The `bbox` coordinate values should be within 0-1000 range.r   dim)r'   r(   
IndexErrorr)   r*   r2   cat)
r6   bboxZleft_position_embeddingsZupper_position_embeddingsZright_position_embeddingsZlower_position_embeddingser)   r*   spatial_position_embeddingsr:   r:   r;   !_calc_spatial_position_embeddingsH   s,    
22z6LayoutLMv2Embeddings._calc_spatial_position_embeddings)__name__
__module____qualname____doc__r"   rD   __classcell__r:   r:   r8   r;   r   3   s    r   c                       s@   e Zd Z fddZdd Zdd Z					dd	d
Z  ZS )LayoutLMv2SelfAttentionc                    s  t    |j|j dkrt|dstd|j d|j d|j| _|j| _t|j|j | _| j| j | _	|j
| _
|j| _|jrhtj|jd| j	 dd| _ttd	d	| j	| _ttd	d	| j	| _nt|j| j	| _t|j| j	| _t|j| j	| _t|j| _d S )
Nr   Zembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r	   Fbiasr   )r!   r"   r$   num_attention_headshasattr
ValueErrorfast_qkvintattention_head_sizeall_head_sizehas_relative_attention_biashas_spatial_attention_biasr   Linear
qkv_linear	Parameterr2   zerosq_biasv_biasquerykeyvaluer.   Zattention_probs_dropout_probr0   r5   r8   r:   r;   r"   c   s*   

z LayoutLMv2SelfAttention.__init__c                 C   s6   |  d d | j| jf }|j| }|ddddS )Nr   r   r<   r   r	   )sizerN   rS   viewpermute)r6   xZnew_x_shaper:   r:   r;   transpose_for_scores}   s   
z,LayoutLMv2SelfAttention.transpose_for_scoresc                 C   s   | j rB| |}tj|ddd\}}}| | j kr'|| j }|| j }n*d| d  d }|| jj|  }|| jj|  }n| |}| 	|}| 
|}|||fS )Nr	   r   r=   r   r   )r   )rQ   rX   r2   chunkZ
ndimensionr[   r\   ra   r]   r^   r_   )r6   hidden_statesZqkvqkvZ_szr:   r:   r;   compute_qkv   s   





z#LayoutLMv2SelfAttention.compute_qkvNFc                 C   s   |  |\}}}	| |}
| |}| |	}|
t| j }
t|
|dd}| jr0||7 }| j	r7||7 }|
 |tjt|jj}tjj|dtjd|}| |}|d urb|| }t||}|dddd }| d d | jf }|j| }|r||f}|S |f}|S )Nr   )r>   dtyper   r<   r   r	   )rk   rd   mathsqrtrS   r2   matmul	transposerU   rV   floatZmasked_fill_toboolfinform   minr   Z
functionalZsoftmaxZfloat32Ztype_asr0   rb   
contiguousr`   rT   ra   )r6   rg   attention_mask	head_maskoutput_attentionsrel_pos
rel_2d_posrh   ri   rj   Zquery_layerZ	key_layerZvalue_layerZattention_scoresZattention_probsZcontext_layerZnew_context_layer_shapeoutputsr:   r:   r;   forward   s2   	




zLayoutLMv2SelfAttention.forwardNNFNN)rE   rF   rG   r"   rd   rk   r~   rI   r:   r:   r8   r;   rJ   b   s    rJ   c                       s0   e Zd Z fddZ					dddZ  ZS )LayoutLMv2Attentionc                    s"   t    t|| _t|| _d S N)r!   r"   rJ   r6   LayoutLMv2SelfOutputoutputr5   r8   r:   r;   r"      s   

zLayoutLMv2Attention.__init__NFc           
      C   s<   | j ||||||d}| |d |}|f|dd   }	|	S )Nr{   r|   r   r   )r6   r   )
r6   rg   rx   ry   rz   r{   r|   Zself_outputsattention_outputr}   r:   r:   r;   r~      s   	zLayoutLMv2Attention.forwardr   rE   rF   rG   r"   r~   rI   r:   r:   r8   r;   r      s    r   c                       $   e Zd Z fddZdd Z  ZS )r   c                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr   )r!   r"   r   rW   r$   denser,   r-   r.   r/   r0   r5   r8   r:   r;   r"         
zLayoutLMv2SelfOutput.__init__c                 C   &   |  |}| |}| || }|S r   r   r0   r,   r6   rg   input_tensorr:   r:   r;   r~         

zLayoutLMv2SelfOutput.forwardr   r:   r:   r8   r;   r      s    r   c                       s2   e Zd Z fddZdejdejfddZ  ZS )LayoutLMv2Intermediatec                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S r   )r!   r"   r   rW   r$   intermediate_sizer   
isinstanceZ
hidden_actstrr
   intermediate_act_fnr5   r8   r:   r;   r"      s
   
zLayoutLMv2Intermediate.__init__rg   returnc                 C   s   |  |}| |}|S r   )r   r   )r6   rg   r:   r:   r;   r~      s   

zLayoutLMv2Intermediate.forwardrE   rF   rG   r"   r2   Tensorr~   rI   r:   r:   r8   r;   r      s    r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )LayoutLMv2Outputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S r   )r!   r"   r   rW   r   r$   r   r,   r-   r.   r/   r0   r5   r8   r:   r;   r"      r   zLayoutLMv2Output.__init__rg   r   r   c                 C   r   r   r   r   r:   r:   r;   r~     r   zLayoutLMv2Output.forwardr   r:   r:   r8   r;   r      s    $r   c                       s8   e Zd Z fddZ					d	ddZdd Z  ZS )
LayoutLMv2Layerc                    s:   t    |j| _d| _t|| _t|| _t|| _	d S )Nr   )
r!   r"   chunk_size_feed_forwardseq_len_dimr   	attentionr   intermediater   r   r5   r8   r:   r;   r"   	  s   


zLayoutLMv2Layer.__init__NFc                 C   sL   | j ||||||d}|d }|dd  }	t| j| j| j|}
|
f|	 }	|	S )N)rz   r{   r|   r   r   )r   r   feed_forward_chunkr   r   )r6   rg   rx   ry   rz   r{   r|   Zself_attention_outputsr   r}   layer_outputr:   r:   r;   r~     s   	
zLayoutLMv2Layer.forwardc                 C   s   |  |}| ||}|S r   )r   r   )r6   r   Zintermediate_outputr   r:   r:   r;   r   -  s   
z"LayoutLMv2Layer.feed_forward_chunkr   )rE   rF   rG   r"   r~   r   rI   r:   r:   r8   r;   r     s    
r   T       c           	      C   s   d}|r|d }|| dk  | 7 }t| }n
t|  t| }|d }||k }|t| | t||  ||  tj  }t	|t
||d }|t|||7 }|S )a  
    Adapted from Mesh Tensorflow:
    https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593
    Translate relative position to a bucket number for relative attention. The relative position is defined as
    memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
    position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for small
    absolute relative_position and larger buckets for larger absolute relative_positions. All relative positions
    >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should
    allow for more graceful generalization to longer sequences than the model has been trained on.

    Args:
        relative_position: an int32 Tensor
        bidirectional: a boolean - whether the attention is bidirectional
        num_buckets: an integer
        max_distance: an integer

    Returns:
        a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
    r   r<   r   )longr2   absmax
zeros_likelogrr   rn   rs   rv   Z	full_likewhere)	Zrelative_positionbidirectionalnum_bucketsmax_distanceretnZ	max_exactZis_smallZval_if_larger:   r:   r;   relative_position_bucket3  s   &r   c                       sD   e Zd Z fddZdd Zdd Z								dd
dZ  ZS )LayoutLMv2Encoderc                    s   t     | _t fddt jD | _ j| _ j	| _	| jr6 j
| _
 j| _tj| j
 jdd| _| j	rW j| _ j| _tj| j jdd| _tj| j jdd| _d| _d S )Nc                    s   g | ]}t  qS r:   )r   ).0_r7   r:   r;   
<listcomp>c  s    z.LayoutLMv2Encoder.__init__.<locals>.<listcomp>FrL   )r!   r"   r7   r   Z
ModuleListrangenum_hidden_layerslayerrU   rV   rel_pos_binsmax_rel_posrW   rN   rel_pos_biasmax_rel_2d_posrel_2d_pos_binsrel_pos_x_biasrel_pos_y_biasgradient_checkpointingr5   r8   r   r;   r"   `  s   
 
zLayoutLMv2Encoder.__init__c                 C   sv   | d| d }t|| j| jd}t  | jj | 	dddd}W d    n1 s0w   Y  |
 }|S )Nrl   r   r   r   r   r	   r   r<   )	unsqueezer   r   r   r2   no_gradr   weighttrb   rw   )r6   r   Zrel_pos_matr{   r:   r:   r;   !_calculate_1d_position_embeddingsu  s   
	z3LayoutLMv2Encoder._calculate_1d_position_embeddingsc           	      C   s   |d d d d df }|d d d d df }| d| d }| d| d }t|| j| jd}t|| j| jd}t $ | jj | 	dddd}| j
j | 	dddd}W d    n1 sgw   Y  | }| }|| }|S )Nr   r	   rl   r   r   r   r<   )r   r   r   r   r2   r   r   r   r   rb   r   rw   )	r6   rA   Zposition_coord_xZposition_coord_yZrel_pos_x_2d_matZrel_pos_y_2d_matZ	rel_pos_xZ	rel_pos_yr|   r:   r:   r;   !_calculate_2d_position_embeddings  s,   
	z3LayoutLMv2Encoder._calculate_2d_position_embeddingsNFTc	              
   C   s  |rdnd }	|r
dnd }
| j r| |nd }| jr| |nd }t| jD ]@\}}|r0|	|f }	|d ur8|| nd }| jrN| jrN| j|j	||||||d}n
|||||||d}|d }|re|
|d f }
q%|rm|	|f }	|s{t
dd ||	|
fD S t||	|
dS )Nr:   r   r   r   c                 s   s    | ]	}|d ur|V  qd S r   r:   )r   rj   r:   r:   r;   	<genexpr>  s    z,LayoutLMv2Encoder.forward.<locals>.<genexpr>)last_hidden_staterg   
attentions)rU   r   rV   r   	enumerater   r   ZtrainingZ_gradient_checkpointing_func__call__tupler   )r6   rg   rx   ry   rz   output_hidden_statesreturn_dictrA   r   Zall_hidden_statesZall_self_attentionsr{   r|   iZlayer_moduleZlayer_head_maskZlayer_outputsr:   r:   r;   r~     sX   

	
	zLayoutLMv2Encoder.forward)NNFFTNN)rE   rF   rG   r"   r   r   r~   rI   r:   r:   r8   r;   r   _  s    r   c                   @   s   e Zd ZeZdZdd ZdS )LayoutLMv2PreTrainedModel
layoutlmv2c                 C   s  t |tjr |jjjd| jjd |jdur|jj	  dS dS t |tj
rC|jjjd| jjd |jdurA|jj|j 	  dS dS t |tjrX|jj	  |jjd dS t |trq| jjro|jj	  |jj	  dS dS t |trt|dr|jjjd| jjd dS dS dS )zInitialize the weightsg        )meanZstdN      ?visual_segment_embedding)r   r   rW   r   dataZnormal_r7   Zinitializer_rangerM   Zzero_r#   r   r,   Zfill_rJ   rQ   r[   r\   LayoutLMv2ModelrO   r   )r6   moduler:   r:   r;   _init_weights  s.   




z'LayoutLMv2PreTrainedModel._init_weightsN)rE   rF   rG   r   Zconfig_classZbase_model_prefixr   r:   r:   r:   r;   r     s    r   c                 C   s   t | tjjjjrtjj| |S | }t | tj	j
rKtjj| j| jdd|d}tj| j|_tj| j|_| j|_| j|_tjdtj| jjd|_|  D ]\}}||t|| qO~ |S )NT)num_featuresr   ZaffineZtrack_running_statsprocess_groupr   rm   device)r   r2   r   modulesZ	batchnormZ
_BatchNormZSyncBatchNormZconvert_sync_batchnorm
detectron2layersFrozenBatchNorm2dr   r   rY   r   rM   Zrunning_meanZrunning_vartensorr   r   Znum_batches_trackedZnamed_childrenZ
add_modulemy_convert_sync_batchnorm)r   r   Zmodule_outputnamechildr:   r:   r;   r      s(   r   c                       s,   e Zd Z fddZdd Zdd Z  ZS )LayoutLMv2VisualBackbonec              	      s  t    | | _| jjj}t|| j}t|j	t
jj	js"J |j	| _	t| jjjt| jjjks6J t| jjj}| jdt| jjj|dddd | jdt| jjj|dddd d| _t rtd d}| j	 | j j}ttt|d	 | |jd	  tt|d | |jd  f| _nt |jd d
 | _t|jd
kr|j!| j	 | j j" | j	 | j j"|jd
 ksJ d S )N
pixel_meanr   Fr   	pixel_stdp2z0using `AvgPool2d` instead of `AdaptiveAvgPool2d`)   r   r   r<   )#r!   r"   Zget_detectron2_configcfgZMODELZMETA_ARCHITECTUREr   getr   backboner   ZmodelingZFPNlenZ
PIXEL_MEANZ	PIXEL_STDr1   r2   r   ra   out_feature_keyZ$are_deterministic_algorithms_enabledloggerwarningZoutput_shapeZstrider   Z	AvgPool2drn   ceilimage_feature_pool_shapepoolZAdaptiveAvgPool2dappendZchannels)r6   r7   Z	meta_archmodelZnum_channelsinput_shapeZbackbone_strider8   r:   r;   r"     s<   


 
  $z!LayoutLMv2VisualBackbone.__init__c                 C   sV   t |r|n|j| j | j }| |}|| j }| |jdd	dd
 }|S )Nr<   )Z	start_dimr   )r2   Z	is_tensorr   r   r   r   r   r   flattenrq   rw   )r6   ZimagesZimages_inputfeaturesr:   r:   r;   r~   <  s
    

z LayoutLMv2VisualBackbone.forwardc                    s   t j rt j rt j dkstdt j }t j t j }| dks.tdfddt	| D   fddt	| D }| }t
| j|| d| _d S )	Nr   z/Make sure torch.distributed is set up properly.r   zGMake sure the number of processes can be divided by the number of nodesc                    s&   g | ]}t t|  |d    qS re   )listr   r   r   )	node_sizer:   r;   r   Q  s   & zCLayoutLMv2VisualBackbone.synchronize_batch_norm.<locals>.<listcomp>c                    s   g | ]}t jj | d qS ))Zranks)r2   distributedZ	new_groupr   )node_global_ranksr:   r;   r   R  s    )r   )r2   r   Zis_availableZis_initializedZget_rankRuntimeErrorcudaZdevice_countZget_world_sizer   r   r   )r6   Z	self_rankZ
world_sizeZsync_bn_groupsZ	node_rankr:   )r   r   r;   synchronize_batch_normC  s"   




z/LayoutLMv2VisualBackbone.synchronize_batch_norm)rE   rF   rG   r"   r~   r   rI   r:   r:   r8   r;   r     s    #r   c                       r   )LayoutLMv2Poolerc                    s*   t    t|j|j| _t | _d S r   )r!   r"   r   rW   r$   r   ZTanh
activationr5   r8   r:   r;   r"   [  s   
zLayoutLMv2Pooler.__init__c                 C   s(   |d d df }|  |}| |}|S )Nr   )r   r   )r6   rg   Zfirst_token_tensorpooled_outputr:   r:   r;   r~   `  s   

zLayoutLMv2Pooler.forwardr   r:   r:   r8   r;   r   Z  s    r   c                       s   e Zd Z fddZdd Zdd Zddd	Zd
d Zdd ZdddZ	e
											d deej deej deej deej deej deej deej deej dee dee dee deeef fddZ  ZS )!r   c                    s   t | d t | || _|j| _t|| _t|| _t	
|jd |j| _| jr8t	t	d|jjd | _t	j|j|jd| _t	|j| _t|| _t|| _|   d S )Nr   r   r   r   r   )r   r!   r"   r7   has_visual_segment_embeddingr   
embeddingsr   visualr   rW   r   r$   visual_projrY   r#   r   r   r,   r-   visual_LayerNormr.   r/   visual_dropoutr   encoderr   pooler	post_initr5   r8   r:   r;   r"   k  s   




zLayoutLMv2Model.__init__c                 C   s   | j jS r   r  r%   r6   r:   r:   r;   get_input_embeddings  s   z$LayoutLMv2Model.get_input_embeddingsc                 C   s   || j _d S r   r  )r6   r_   r:   r:   r;   set_input_embeddings  s   z$LayoutLMv2Model.set_input_embeddingsNc                 C   s   |d ur	|  }n|  d d }|d }|d u r+tj|tj|jd}|d|}|d u r4t|}|d u r>| j	|}| j
|}| j|}	| j|}
|| |	 |
 }| j|}| j|}|S )Nr   r   r   r   )r`   r2   r3   r   r   r   Z	expand_asr   r  r%   r&   rD   r+   r,   r0   )r6   	input_idsrA   r   token_type_idsinputs_embedsr   
seq_lengthr&   rC   r+   r  r:   r:   r;   _calc_text_embeddings  s$   

z%LayoutLMv2Model._calc_text_embeddingsc                 C   s\   |  | |}| j|}| j|}|| | }| jr"|| j7 }| |}| |}|S r   )	r  r  r  r&   rD   r  r   r  r  )r6   imagerA   r   Zvisual_embeddingsr&   rC   r  r:   r:   r;   _calc_img_embeddings  s   


z$LayoutLMv2Model._calc_img_embeddingsc                 C   s
  t jt jdd|d d  d||jd| jjd dd}t jt jdd| jjd d  d||jd| jjd dd}t j|d d |d d|d d |d ddd|dd  |d d|dd  |d dddgdd	d|
d}||d dd}|S )	Nr   i  r   )r   rm   floor)Zrounding_moder   r=   )r2   divr3   rm   r7   r   stackrepeatrq   ra   r`   )r6   r   rA   r   final_shapeZvisual_bbox_xZvisual_bbox_yvisual_bboxr:   r:   r;   _calc_visual_bbox  sD   


z!LayoutLMv2Model._calc_visual_bboxc                 C   sH   |d ur|d urt d|d ur| S |d ur | d d S t d)NDYou cannot specify both input_ids and inputs_embeds at the same timer   5You have to specify either input_ids or inputs_embeds)rP   r`   )r6   r  r  r:   r:   r;   _get_input_shape  s   z LayoutLMv2Model._get_input_shaper  rA   r  rx   r  r   ry   r  rz   r   r   r   c              
   C   s  |	dur|	n| j j}	|
dur|
n| j j}
|dur|n| j j}| ||}|dur+|jn|j}t|}| j jd | j jd  |d< t	|}t| ||}|d  |d 7  < t	|}| 
| j j|||}tj||gdd}|du rztj||d}tj||d}tj||gdd}|du rtj|tj|d}|du r|d }| jjddd|f }||}tjd|d tj|d|d d}tj||gdd}|du rtjtt|dg tj|d}| j|||||d}| j|||d	}tj||gdd}|dd
}|j| jd}d| t| jj }|durX| dkr:|dddd}|| j jdddd}n| d
krL|ddd}|jt|  jd}ndg| j j }| j ||||||	|
|d}|d }| !|}|s||f|dd  S t"|||j#|j$dS )aK  
        bbox (`torch.LongTensor` of shape `((batch_size, sequence_length), 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.
        image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`):
            Batch of document images.

        Examples:

        ```python
        >>> from transformers import AutoProcessor, LayoutLMv2Model, set_seed
        >>> from PIL import Image
        >>> import torch
        >>> from datasets import load_dataset

        >>> set_seed(0)

        >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased")
        >>> model = LayoutLMv2Model.from_pretrained("microsoft/layoutlmv2-base-uncased")


        >>> dataset = load_dataset("hf-internal-testing/fixtures_docvqa", trust_remote_code=True)
        >>> image_path = dataset["test"][0]["file"]
        >>> image = Image.open(image_path).convert("RGB")

        >>> encoding = processor(image, return_tensors="pt")

        >>> outputs = model(**encoding)
        >>> last_hidden_states = outputs.last_hidden_state

        >>> last_hidden_states.shape
        torch.Size([1, 342, 768])
        ```
        Nr   r   r=   )r   r      )r  rA   r  r   r  r  rA   r   r<   )rm   r   r   )rA   r   ry   rz   r   r   )r   Zpooler_outputrg   r   )%r7   rz   r   use_return_dictr  r   r   r   r2   Sizer  r@   ZonesrZ   r   r  r   r4   r3   r  r   r  r  r   rs   rm   ru   rv   r>   r   next
parametersr  r	  r   rg   r   )r6   r  rA   r  rx   r  r   ry   r  rz   r   r   r   r   visual_shaper  r  Z
final_bboxZvisual_attention_maskZfinal_attention_maskr  visual_position_idsZfinal_position_idsZtext_layout_embZ
visual_embZ	final_embZextended_attention_maskZencoder_outputssequence_outputr  r:   r:   r;   r~     s   3


 


zLayoutLMv2Model.forwardr   )NN)NNNNNNNNNNN)rE   rF   rG   r"   r  r  r  r  r  r  r   r   r2   
LongTensorFloatTensorrt   r   r   r   r~   rI   r:   r:   r8   r;   r   i  sZ    

%
	

r   ax  
    LayoutLMv2 Model with a sequence classification head on top (a linear layer on top of the concatenation of the
    final hidden state of the [CLS] token, average-pooled initial visual embeddings and average-pooled final visual
    embeddings, e.g. for document image classification tasks such as the
    [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset.
    )Zcustom_introc                          e Zd Z fddZdd Ze												ddeej deej deej	 d	eej	 d
eej deej deej	 deej	 deej dee
 dee
 dee
 deeef fddZ  ZS )#LayoutLMv2ForSequenceClassificationc                    sN   t  | |j| _t|| _t|j| _t	|j
d |j| _|   d S )Nr	   r!   r"   
num_labelsr   r   r   r.   r/   r0   rW   r$   
classifierr
  r5   r8   r:   r;   r"   t  s   
z,LayoutLMv2ForSequenceClassification.__init__c                 C   
   | j jjS r   r   r  r%   r  r:   r:   r;   r  ~     
z8LayoutLMv2ForSequenceClassification.get_input_embeddingsNr  rA   r  rx   r  r   ry   r  labelsrz   r   r   r   c                 C   s0  |dur|n| j j}|dur|durtd|dur%| || | }n|dur2| dd }ntd|dur=|jn|j}t|}| j jd | j jd  |d< t	|}t|}|d  |d 7  < t	|}| j
| j j|||}tjd|d tj|d|d d}| j
j|||d}| j
|||||||||
||d	}|dur| }n| dd }|d }|d ddd|f |d dd|df }}|dddddf }|jdd
}|jdd
}tj|||gdd
}| |}| |}d}|	durv| j jdu r/| jdkrd| j _n| jdkr+|	jtjks&|	jtjkr+d| j _nd| j _| j jdkrOt }| jdkrI|| |	 }n-|||	}n'| j jdkrgt }||d| j|	d}n| j jdkrvt }|||	}|s|f|dd  }|dur|f| S |S t|||j|jdS )a  
        input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.
        image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`):
            Batch of document images.
        token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Example:

        ```python
        >>> from transformers import AutoProcessor, LayoutLMv2ForSequenceClassification, set_seed
        >>> from PIL import Image
        >>> import torch
        >>> from datasets import load_dataset

        >>> set_seed(0)

        >>> dataset = load_dataset("aharley/rvl_cdip", split="train", streaming=True, trust_remote_code=True)
        >>> data = next(iter(dataset))
        >>> image = data["image"].convert("RGB")

        >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased")
        >>> model = LayoutLMv2ForSequenceClassification.from_pretrained(
        ...     "microsoft/layoutlmv2-base-uncased", num_labels=dataset.info.features["label"].num_classes
        ... )

        >>> encoding = processor(image, return_tensors="pt")
        >>> sequence_label = torch.tensor([data["label"]])

        >>> outputs = model(**encoding, labels=sequence_label)

        >>> loss, logits = outputs.loss, outputs.logits
        >>> predicted_idx = logits.argmax(dim=-1).item()
        >>> predicted_answer = dataset.info.features["label"].names[4]
        >>> predicted_idx, predicted_answer  # results are not good without further fine-tuning
        (7, 'advertisement')
        ```
        Nr  r   r  r   r   r   r!  r  rA   r  rx   r  r   ry   r  rz   r   r   r=   Z
regressionZsingle_label_classificationZmulti_label_classificationr<   losslogitsrg   r   ) r7   r"  rP   Z%warn_if_padding_and_no_attention_maskr`   r   r   r   r2   r#  r   r  r3   r   r  r  r   r@   r0   r/  Zproblem_typer.  rm   rR   r   squeezer   ra   r   r   rg   r   )r6   r  rA   r  rx   r  r   ry   r  r3  rz   r   r   r   r   r&  r  r  r'  Zinitial_image_embeddingsr}   r  r(  Zfinal_image_embeddingsZcls_final_outputZpooled_initial_image_embeddingsZpooled_final_image_embeddingsr7  r6  loss_fctr   r:   r:   r;   r~     s   Q



2




(

z+LayoutLMv2ForSequenceClassification.forwardNNNNNNNNNNNN)rE   rF   rG   r"   r  r   r   r2   r)  r*  rt   r   r   r   r~   rI   r:   r:   r8   r;   r,  k  V    	
	

r,  a  
    LayoutLMv2 Model with a token classification head on top (a linear layer on top of the text part of the hidden
    states) e.g. for sequence labeling (information extraction) tasks such as
    [FUNSD](https://guillaumejaume.github.io/FUNSD/), [SROIE](https://rrc.cvc.uab.es/?ch=13),
    [CORD](https://github.com/clovaai/cord) and [Kleister-NDA](https://github.com/applicaai/kleister-nda).
    c                       r+  ) LayoutLMv2ForTokenClassificationc                    sJ   t  | |j| _t|| _t|j| _t	|j
|j| _|   d S r   r-  r5   r8   r:   r;   r"   A  s   
z)LayoutLMv2ForTokenClassification.__init__c                 C   r0  r   r1  r  r:   r:   r;   r  K  r2  z5LayoutLMv2ForTokenClassification.get_input_embeddingsNr  rA   r  rx   r  r   ry   r  r3  rz   r   r   r   c                 C   s   |dur|n| j j}| j|||||||||
||d}|dur#| }n| dd }|d }|d ddd|f }| |}| |}d}|	dur[t }||d| j|	d}|sq|f|dd  }|duro|f| S |S t	|||j
|jdS )a~  
        input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.
        image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`):
            Batch of document images.
        token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, LayoutLMv2ForTokenClassification, set_seed
        >>> from PIL import Image
        >>> from datasets import load_dataset

        >>> set_seed(0)

        >>> datasets = load_dataset("nielsr/funsd", split="test", trust_remote_code=True)
        >>> labels = datasets.features["ner_tags"].feature.names
        >>> id2label = {v: k for v, k in enumerate(labels)}

        >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased", revision="no_ocr")
        >>> model = LayoutLMv2ForTokenClassification.from_pretrained(
        ...     "microsoft/layoutlmv2-base-uncased", num_labels=len(labels)
        ... )

        >>> data = datasets[0]
        >>> image = Image.open(data["image_path"]).convert("RGB")
        >>> words = data["words"]
        >>> boxes = data["bboxes"]  # make sure to normalize your bounding boxes
        >>> word_labels = data["ner_tags"]
        >>> encoding = processor(
        ...     image,
        ...     words,
        ...     boxes=boxes,
        ...     word_labels=word_labels,
        ...     padding="max_length",
        ...     truncation=True,
        ...     return_tensors="pt",
        ... )

        >>> outputs = model(**encoding)
        >>> logits, loss = outputs.logits, outputs.loss

        >>> predicted_token_class_ids = logits.argmax(-1)
        >>> predicted_tokens_classes = [id2label[t.item()] for t in predicted_token_class_ids[0]]
        >>> predicted_tokens_classes[:5]  # results are not good without further fine-tuning
        ['I-HEADER', 'I-HEADER', 'I-QUESTION', 'I-HEADER', 'I-QUESTION']
        ```
        Nr4  r   r   r   r<   r5  )r7   r"  r   r`   r0   r/  r   ra   r.  r   rg   r   )r6   r  rA   r  rx   r  r   ry   r  r3  rz   r   r   r}   r   r  r(  r7  r6  r9  r   r:   r:   r;   r~   N  sD   Z


z(LayoutLMv2ForTokenClassification.forwardr:  )rE   rF   rG   r"   r  r   r   r2   r)  r*  rt   r   r   r   r~   rI   r:   r:   r8   r;   r<  8  r;  r<  c                        s   e Zd Zd fdd	Zdd Ze													ddeej deej d	eej	 d
eej	 deej deej deej	 deej	 deej deej dee
 dee
 dee
 deeef fddZ  ZS )LayoutLMv2ForQuestionAnsweringTc                    sB   t  | |j| _||_t|| _t|j|j| _	| 
  dS )z
        has_visual_segment_embedding (`bool`, *optional*, defaults to `True`):
            Whether or not to add visual segment embeddings.
        N)r!   r"   r.  r  r   r   r   rW   r$   
qa_outputsr
  )r6   r7   r  r8   r:   r;   r"     s   
z'LayoutLMv2ForQuestionAnswering.__init__c                 C   r0  r   r1  r  r:   r:   r;   r    r2  z3LayoutLMv2ForQuestionAnswering.get_input_embeddingsNr  rA   r  rx   r  r   ry   r  start_positionsend_positionsrz   r   r   r   c                 C   s  |dur|n| j j}| j|||||||||||d}|dur#| }n| dd }|d }|d ddd|f }| |}|jddd\}}|d }|d }d}|	dur|
durt|	 dkrn|	d}	t|
 dkr{|
d}
|d}|		d|}	|
	d|}
t
|d}|||	}|||
}|| d }|s||f|dd  }|dur|f| S |S t||||j|jd	S )
u  
        input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner.
        image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`):
            Batch of document images.
        token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)

        Example:

        In this example below, we give the LayoutLMv2 model an image (of texts) and ask it a question. It will give us
        a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image).

        ```python
        >>> from transformers import AutoProcessor, LayoutLMv2ForQuestionAnswering, set_seed
        >>> import torch
        >>> from PIL import Image
        >>> from datasets import load_dataset

        >>> set_seed(0)
        >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased")
        >>> model = LayoutLMv2ForQuestionAnswering.from_pretrained("microsoft/layoutlmv2-base-uncased")

        >>> dataset = load_dataset("hf-internal-testing/fixtures_docvqa", trust_remote_code=True)
        >>> image_path = dataset["test"][0]["file"]
        >>> image = Image.open(image_path).convert("RGB")
        >>> question = "When is coffee break?"
        >>> encoding = processor(image, question, return_tensors="pt")

        >>> outputs = model(**encoding)
        >>> predicted_start_idx = outputs.start_logits.argmax(-1).item()
        >>> predicted_end_idx = outputs.end_logits.argmax(-1).item()
        >>> predicted_start_idx, predicted_end_idx
        (30, 191)

        >>> predicted_answer_tokens = encoding.input_ids.squeeze()[predicted_start_idx : predicted_end_idx + 1]
        >>> predicted_answer = processor.tokenizer.decode(predicted_answer_tokens)
        >>> predicted_answer  # results are not good without further fine-tuning
        '44 a. m. to 12 : 25 p. m. 12 : 25 to 12 : 58 p. m. 12 : 58 to 4 : 00 p. m. 2 : 00 to 5 : 00 p. m. coffee break coffee will be served for men and women in the lobby adjacent to exhibit area. please move into exhibit area. ( exhibits open ) trrf general session ( part | ) presiding : lee a. waller trrf vice president “ introductory remarks ” lee a. waller, trrf vice presi - dent individual interviews with trrf public board members and sci - entific advisory council mem - bers conducted by trrf treasurer philip g. kuehn to get answers which the public refrigerated warehousing industry is looking for. plus questions from'
        ```

        ```python
        >>> target_start_index = torch.tensor([7])
        >>> target_end_index = torch.tensor([14])
        >>> outputs = model(**encoding, start_positions=target_start_index, end_positions=target_end_index)
        >>> predicted_answer_span_start = outputs.start_logits.argmax(-1).item()
        >>> predicted_answer_span_end = outputs.end_logits.argmax(-1).item()
        >>> predicted_answer_span_start, predicted_answer_span_end
        (30, 191)
        ```
        Nr4  r   r   r   r=   )Zignore_indexr<   )r6  start_logits
end_logitsrg   r   )r7   r"  r   r`   r>  splitr8  rw   r   clampr   r   rg   r   )r6   r  rA   r  rx   r  r   ry   r  r?  r@  rz   r   r   r}   r   r  r(  r7  rA  rB  Z
total_lossZignored_indexr9  Z
start_lossZend_lossr   r:   r:   r;   r~     s\   Z







z&LayoutLMv2ForQuestionAnswering.forward)T)NNNNNNNNNNNNN)rE   rF   rG   r"   r  r   r   r2   r)  r*  rt   r   r   r   r~   rI   r:   r:   r8   r;   r=    s\    	

r=  )r=  r,  r<  r   r   r   )Tr   r   r   )=rH   rn   typingr   r   r   r2   Ztorch.utils.checkpointr   Ztorch.nnr   r   r   Zactivationsr
   Zmodeling_outputsr   r   r   r   r   Zmodeling_utilsr   Zpytorch_utilsr   utilsr   r   r   r   Zconfiguration_layoutlmv2r   r   Zdetectron2.modelingr   ModuleZ_load_from_state_dictr   Z
batch_normr   Z
get_loggerrE   r   r   rJ   r   r   r   r   r   r   r   r   r   r   r   r   r,  r<  r=  __all__r:   r:   r:   r;   <module>   sf   
/]
+, 
B   F  )