o
    Zhs0                    @  s  d Z ddlmZ ddlZddlmZ ddlmZmZm	Z	m
Z
 ddlZddlZddlmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZ dd	lm Z m!Z!m"Z" dd
l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ e(,e-Z.dZ/dZ0G dd dej1j2Z3G dd dej1j2Z4G dd dej1j2Z5G dd dej1j2Z6eG dd dej1j2Z7G dd deZ8eG dd de$Z9eG dd de$Z:eG dd  d e$Z;eG d!d" d"e$Z<eG d#d$ d$e$Z=eG d%d& d&e$Z>d'Z?d(Z@e&d)e?G d*d+ d+e8ZAe&d,e?G d-d. d.e8eZBe&d/e?G d0d1 d1e8eZCe&d2e?G d3d4 d4e8eZDe&d5e?G d6d7 d7e8eZEe&d8e?G d9d: d:e8eZFg d;ZGdS )<z
TF 2.0 XLNet model.
    )annotationsN)	dataclass)ListOptionalTupleUnion   )get_tf_activation)TFCausalLanguageModelingLossTFModelInputTypeTFMultipleChoiceLossTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFSequenceSummaryTFSharedEmbeddingsTFTokenClassificationLossget_initializerkeraskeras_serializableunpack_inputs)check_embeddings_within_bounds
shape_liststable_softmax)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )XLNetConfigzxlnet/xlnet-base-casedr!   c                      sd   e Zd Z fddZdddZdd Zdd	d
Z	dddZdddZ					dd ddZ	  Z
S )!TFXLNetRelativeAttentionc                   s   t  jdi | |j|j dkrtd|j d|j |j| _|j| _|j| _d|jd  | _|j| _|j| _t	j
j|jdd| _t	j
|j| _|| _d S )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (r    g      ?
layer_normepsilonname )super__init__d_modeln_head
ValueErrord_headscaleinitializer_rangeoutput_attentionsr   layersLayerNormalizationlayer_norm_epsr#   Dropoutdropoutconfigselfr6   kwargs	__class__r'   Z/var/www/auris/lib/python3.10/site-packages/transformers/models/xlnet/modeling_tf_xlnet.pyr)   @   s    

z!TFXLNetRelativeAttention.__init__Nc                 C  s  t | j}| j| j| j| jf|ddd| _| j| j| j| jf|ddd| _| j| j| j| jf|ddd| _| j| j| j| jf|ddd| _	| j| j| j| jf|ddd| _
| j| j| jfddd	d| _| j| j| jfddd
d| _| j| j| jfdddd| _| jd| j| jf|ddd| _| jrd S d| _t| dd d urt| jj | jd d | jjg W d    d S 1 sw   Y  d S d S )NTqshapeinitializerZ	trainabler&   kvorzerosr_r_biasr_s_biasr_w_bias   	seg_embedr#   )r   r/   
add_weightr*   r+   r-   r=   rA   rB   rC   rD   rF   rG   rH   rJ   builtgetattrtf
name_scoper#   r&   buildr6   )r8   input_shaper@   r'   r'   r<   rP   T   sH   
"zTFXLNetRelativeAttention.buildc                 C     t NNotImplementedError)r8   Zheadsr'   r'   r<   prune_headsy      z$TFXLNetRelativeAttention.prune_headsc                 C  s   t |}t||d |d |d |d f}|dddf }t||d |d d |d |d f}|ddd|ddddf }|S )z<perform relative shift to form the relative attention score.r    r   rI   r   N.)r   rN   reshape)r8   xklenZx_sizer'   r'   r<   	rel_shift|   s   $( z"TFXLNetRelativeAttention.rel_shiftFc
                 C  s  t d|| j |}
t d|| j |}| j|t|
d d}|du r&d}nt d|| j | j}t d||}|
| | | j }|dur^|j	t j
ksQ|j	t jkrX|d|  }n|d	|  }t|dd
}| j||	d}|durs|| }t d||}|r||fS |S )z.Core relative positional attention operations.zibnd,jbnd->ijbnr    )r[   Nr   zibnd,snd->ibnszijbs,ibns->ijbni  gꌠ9Y>)FZaxistrainingzijbn,jbnd->ibnd)rN   einsumrH   rF   r\   r   rG   rJ   r.   dtypeZfloat16Zbfloat16r   r5   )r8   Zq_headk_head_hv_head_hk_head_rseg_mat	attn_mask	head_maskr0   r_   acZbdZefZ
attn_score	attn_probattn_vecr'   r'   r<   rel_attn_core   s(   z&TFXLNetRelativeAttention.rel_attn_coreTc                 C  s8   t d|| j}| j||d}|r|| }| |}|S )zPost-attention processing.zibnd,hnd->ibhr^   )rN   r`   rC   r5   r#   )r8   hrj   Zresidualr_   Zattn_outoutputr'   r'   r<   post_attention   s   
z'TFXLNetRelativeAttention.post_attentionmemsnp.ndarray | tf.Tensor | Nonetarget_mappingrg   r0   Optional[bool]r_   boolc                 C  s  |d ur|d urt t|dkrtj||gdd}n|}td|| j}td|| j}td|| j}td|| j}| j	|||||||	|
|d	}|
rP|\}}| j
|||d}td|| j}|d urtd||}| j	|||||||	|
|d	}|
r|\}}td||}n| j	|||||||	|
|d	}|
r|\}}| j
|||d}|
r||f}nV|d urt t|dkrtj||gdd}n|}td|| j}td|| j}td|| j}td|| j}| j	|||||||	|
|d	}|
r|\}}| j
|||d}d }||f}|
r||f }|S )Nr    r   r]   zibh,hnd->ibndr^   zmbnd,mlb->lbndzlbnd,mlb->mbnd)lenr   rN   concatr`   rA   rB   rD   r=   rk   rn   )r8   rl   gZattn_mask_hZattn_mask_grD   re   ro   rq   rg   r0   r_   catrb   rc   rd   Zq_head_hZ
attn_vec_hZattn_prob_houtput_hZq_head_gZ
attn_vec_gZattn_prob_goutput_gri   rj   outputsr'   r'   r<   call   s   
zTFXLNetRelativeAttention.callrS   )rX   F)TFNNNFF
ro   rp   rq   rp   rg   rp   r0   rr   r_   rs   )__name__
__module____qualname__r)   rP   rV   r\   rk   rn   r{   __classcell__r'   r'   r:   r<   r"   ?   s    
%


,r"   c                      s0   e Zd Z fddZd	ddZd
ddZ  ZS )TFXLNetFeedForwardc                   s   t  jdi | tjj|jdd| _tjj|jt	|j
dd| _tjj|jt	|j
dd| _tj|j| _t|jtrDt|j| _n|j| _|| _d S )Nr#   r$   layer_1Zkernel_initializerr&   layer_2r'   )r(   r)   r   r1   r2   r3   r#   Densed_innerr   r/   r   r*   r   r4   r5   
isinstanceZff_activationstrr	   activation_functionr6   r7   r:   r'   r<   r)   N  s   
zTFXLNetFeedForward.__init__Fc                 C  sP   |}|  |}| |}| j||d}| |}| j||d}| || }|S )Nr^   )r   r   r5   r   r#   )r8   inpr_   rm   r'   r'   r<   r{   ^  s   


zTFXLNetFeedForward.callNc                 C  s  | j rd S d| _ t| dd d ur2t| jj | jd d | jjg W d    n1 s-w   Y  t| dd d ur\t| j	j | j	d d | jjg W d    n1 sWw   Y  t| dd d urt| j
j | j
d d | jjg W d    d S 1 sw   Y  d S d S )NTr#   r   r   )rL   rM   rN   rO   r#   r&   rP   r6   r*   r   r   r   r8   rQ   r'   r'   r<   rP   h  s    "zTFXLNetFeedForward.buildr|   rS   r   r   r   r)   r{   rP   r   r'   r'   r:   r<   r   M  s    

r   c                      s<   e Zd Z fddZ					ddddZdddZ  ZS )TFXLNetLayerc                   sB   t  jdi | t|dd| _t|dd| _tj|j	| _	d S )Nrel_attnr&   ffr'   )
r(   r)   r"   r   r   r   r   r1   r4   r5   r7   r:   r'   r<   r)   x  s   zTFXLNetLayer.__init__NFro   rp   rq   rg   r0   rr   r_   rs   c                 C  sl   | j |||||||||	|
|d}|d d \}}|d ur#| j||d}| j||d}||f|dd   }|S )Nr^   rI   )r   r   )r8   rx   ry   non_tgt_maskrf   pos_embre   ro   rq   rg   r0   r_   rz   r'   r'   r<   r{   ~  s&   zTFXLNetLayer.callc                 C     | j rd S d| _ t| dd d ur-t| jj | jd  W d    n1 s(w   Y  t| dd d urUt| jj | jd  W d    d S 1 sNw   Y  d S d S )NTr   r   )rL   rM   rN   rO   r   r&   rP   r   r   r'   r'   r<   rP        "zTFXLNetLayer.buildr}   r~   rS   r   r'   r'   r:   r<   r   w  s    $r   c                      sP   e Zd Z fddZ fddZdd Zdd Zd	d
 Zdd Zdd Z	  Z
S )TFXLNetLMHeadc                   s"   t  jdi | || _|| _d S )Nr'   )r(   r)   r6   input_embeddings)r8   r6   r   r9   r:   r'   r<   r)     s   
zTFXLNetLMHead.__init__c                   s*   | j | jjfdddd| _t | d S )NrE   Tbiasr>   )rK   r6   
vocab_sizer   r(   rP   r   r:   r'   r<   rP     s   zTFXLNetLMHead.buildc                 C     | j S rS   )r   r8   r'   r'   r<   get_output_embeddings     z#TFXLNetLMHead.get_output_embeddingsc                 C     || j _t|d | j _d S Nr   )r   weightr   r   r8   valuer'   r'   r<   set_output_embeddings     z#TFXLNetLMHead.set_output_embeddingsc                 C  s
   d| j iS )Nr   )r   r   r'   r'   r<   get_bias  s   
zTFXLNetLMHead.get_biasc                 C  s"   |d | _ t|d d | j_d S )Nr   r   )r   r   r6   r   r   r'   r'   r<   set_bias  s   
zTFXLNetLMHead.set_biasc                 C  s   | j |dd}|| j }|S )NZlinear)mode)r   r   )r8   hidden_statesr'   r'   r<   r{     s   
zTFXLNetLMHead.call)r   r   r   r)   rP   r   r   r   r   r{   r   r'   r'   r:   r<   r     s    r   c                      s   e Zd ZeZ fddZdd Zdd Zd)dd	Zd
d Z	dd Z
dd Zed)ddZd)ddZe														d*d+d'd(Z  ZS ),TFXLNetMainLayerc                   s   t  jdi |  | _ j| _ j| _ j| _ j| _ j| _ j| _ j	| _	 j
| _
 j| _ j| _ j| _ j| _ j| _t j j jdd| _ fddt jD | _tj j| _ j| _ j| _d S )Nword_embeddingr/   r&   c                   s   g | ]}t  d | dqS )zlayer_._r   )r   ).0ir6   r'   r<   
<listcomp>  s    z-TFXLNetMainLayer.__init__.<locals>.<listcomp>r'   )r(   r)   r6   output_hidden_statesr0   return_dictmem_len	reuse_lenr*   same_length	attn_typebi_data	clamp_lenn_layerZuse_bfloat16r/   r   r   r   rangelayerr   r1   r4   r5   use_mems_evaluse_mems_trainr7   r:   r   r<   r)     s,   zTFXLNetMainLayer.__init__c                 C  r   rS   )r   r   r'   r'   r<   get_input_embeddings  r   z%TFXLNetMainLayer.get_input_embeddingsc                 C  r   r   )r   r   r   r   r   r'   r'   r<   set_input_embeddings  r   z%TFXLNetMainLayer.set_input_embeddingsNc              	   C  s   t | j}| jdd| jf|ddd| _| jrd S d| _t| dd d ur@t| j	j
 | j	d  W d    n1 s;w   Y  t| dd d uri| jD ]}t|j
 |d  W d    n1 scw   Y  qKd S d S )Nr    Tmask_embr>   r   r   )r   r/   rK   r*   r   rL   rM   rN   rO   r   r&   rP   r   )r8   rQ   r@   r   r'   r'   r<   rP     s&   

zTFXLNetMainLayer.buildc                 C  rR   rS   rT   )r8   Zheads_to_pruner'   r'   r<   _prune_heads  rW   zTFXLNetMainLayer._prune_headsc           	      C  s   t ||g}t j|dd}t j|dd}t ||g}t ||| gd}| jrOt j|dd}t |ddd|f | | |dd|df gd}|S )a  
        Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked.

        Args:
            qlen: TODO Lysandre didn't fill
            mlen: TODO Lysandre didn't fill

        ```

                  same_length=False:      same_length=True:
                  <mlen > <  qlen >       <mlen > <  qlen >
               ^ [0 0 0 0 0 1 1 1 1]     [0 0 0 0 0 1 1 1 1]
                 [0 0 0 0 0 0 1 1 1]     [1 0 0 0 0 0 1 1 1]
            qlen [0 0 0 0 0 0 0 1 1]     [1 1 0 0 0 0 0 1 1]
                 [0 0 0 0 0 0 0 0 1]     [1 1 1 0 0 0 0 0 1]
               v [0 0 0 0 0 0 0 0 0]     [1 1 1 1 0 0 0 0 0]
        ```
        r   rX   r    N)rN   onesZlinalgZ	band_partrE   ru   r   )	r8   qlenmlenrf   Zmask_uZmask_diaZattn_mask_padretZmask_lr'   r'   r<   create_mask	  s   8zTFXLNetMainLayer.create_maskc                 C  s|   | j d ur| j dkr|d | j  }| jd u s| jdkrd}n| j }|d u r-||d  }nt||gd|d  }t|S r   )r   r   rN   ru   Zstop_gradient)r8   Zcurr_outZprev_memcutoffZnew_memr'   r'   r<   	cache_mem&  s   
zTFXLNetMainLayer.cache_memc                 C  s`   t d| |}t jt |t |gdd}|d d d d d f }|d ur.t |d|dg}|S )Nzi,d->idrX   r]   r    )rN   r`   ru   sincostile)Zpos_seqinv_freqbszZsinusoid_inpr   r'   r'   r<   positional_embedding;  s   z%TFXLNetMainLayer.positional_embeddingc                 C  sp  t d| jd}dd|| j   }| jdkr|| }}n| jdkr(|d}}n	td| j d	| jrt ||d
}t | | d}	| jdkr]t || j | j}t |	| j | j}	|dur|d dkrotd| d| |||d }
| |	||d }n| ||}
| |	|}t j	|
|gdd}|S t ||d
}| jdkrt || j | j}| |||}|S )z$create relative positional encoding.r   g       @r    i'  biunirX   zUnknown `attn_type` .g            ?NrI   zWith bi_data, the batch size z should be divisible by 2r]   )
rN   r   r*   r   r,   r   r   Zclip_by_valuer   ru   )r8   r   r[   r   Zfreq_seqr   begendZfwd_pos_seqZbwd_pos_seqZfwd_pos_embZbwd_pos_embr   r'   r'   r<   relative_positional_encodingF  s6   



z-TFXLNetMainLayer.relative_positional_encodingF	input_idsTFModelInputType | Noneattention_maskrp   ro   	perm_maskrq   token_type_ids
input_maskrg   inputs_embedsuse_memsrr   r0   r   r   r_   rs   c           '      C  s>  |r
|
d u r
| j }
n| j}
|d ur|	d urtd|d ur/tj|dd}t|d d \}}n|	d urEtj|	dd}	t|	d d \}}ntd|d urTtj|ddnd }|d uratj|ddnd }|d urntj|ddnd }|d ur{tj|ddnd }|d urtj|ddnd }|d ur|d d urt|d d nd}|| }| jd	kr| ||}|d d d d d d f }n| jd
krd }ntd| j |d u s|d u sJ d|d u r|d urtd}dtj	||j
d }|d ur|d ur|d  | }n|d ur|d u r|d  }n|d u r|d ur|}nd }|d ur]|dkr9tt|d ||g}tj||gdd}|d u rM|d d d d d d d f }n||d d d d d d d f 7 }|d urltj	|dk|j
d}|d urt| }|dkrtjt||g|gdd}tj	||d d d d d d f  dk|j
d}nd }|	d ur|	}nt|| jj | |}| j||d}|d urt| jt|d |dg}| j||d}nd }|d ur|dkrtj||g|j
d}t||gd}n|}tj	tt|d d d f |d d d f |j
d}t|d}nd }| j|||d}| j||d}|d ur4td g| j }d} |d u rId gt| j }|rNg nd }!|rUg nd }"t| jD ]I\}#}$|
rn| | |||# f } |r|"|d ur|||fn| |$||||||||# |||# ||d}%|%d d \}}|r|!|%d  q\|r|"|d ur||fn| | j|d ur|n||d}&tj|&dd}&|
sd } |r|d urtdd |"D }"n	tdd |"D }"|r|d urtdd |!D }!n	tdd |!D }!|stdd |&| |"|!fD S t |&| |"|!dS )NzDYou cannot specify both input_ids and inputs_embeds at the same time)r    r   permrI   r    r   rI   z5You have to specify either input_ids or inputs_embeds)r    rI   r   r   r   r   zUnsupported attention type: zYou can only use one of input_mask (uses 1 for padding) or attention_mask (uses 0 for padding, added for compatibility with BERT). Please choose one.r   ra   r    r]   rX   r^   )r   r'   c                 s  s(    | ]}|D ]
}t j|d dV  qqdS r   r   NrN   	transpose)r   hsrl   r'   r'   r<   	<genexpr>(  s   & z(TFXLNetMainLayer.call.<locals>.<genexpr>c                 s      | ]
}t j|d dV  qdS r   r   )r   r   r'   r'   r<   r   *      c                 s  s"    | ]}t d d |D V  qdS )c                 s  r   )rI   r   r   r    r   Nr   )r   Zattn_streamr'   r'   r<   r   /  r   z2TFXLNetMainLayer.call.<locals>.<genexpr>.<genexpr>N)tupler   tr'   r'   r<   r   .  s    
c                 s  r   r   r   r   r'   r'   r<   r   2  r   c                 s  s    | ]	}|d ur|V  qd S rS   r'   )r   rB   r'   r'   r<   r   5  s    )last_hidden_statero   r   
attentions)!r   r   r,   rN   r   r   r   r   Zconstantcastra   rE   ru   eyer   r   r   r5   r   r   Zlogical_notequalZone_hotr   rU   r   rt   r   	enumerater   appendr   TFXLNetModelOutput)'r8   r   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r_   r   r   r   r[   rf   Zone_cstZ	data_maskZ	mems_maskr   Z
word_emb_krx   Z
word_emb_qry   Zmem_padZcat_idsre   r   Znew_memsr   r   r   Zlayer_modulerz   rm   r'   r'   r<   r{   n  s   (






 


.




(




zTFXLNetMainLayer.callrS   NNNNNNNNNNNNNF)r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r_   rs   )r   r   r   r!   config_classr)   r   r   rP   r   r   r   staticmethodr   r   r   r{   r   r'   r'   r:   r<   r     s8    


(r   c                   @  s   e Zd ZdZeZdZdS )TFXLNetPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    transformerN)r   r   r   __doc__r!   r   Zbase_model_prefixr'   r'   r'   r<   r   <  s    r   c                   @  sB   e Zd ZU dZdZded< dZded< dZded< dZded	< dS )
r   a  
    Output type of [`TFXLNetModel`].

    Args:
        last_hidden_state (`tf.Tensor` of shape `(batch_size, num_predict, hidden_size)`):
            Sequence of hidden-states at the last layer of the model.

            `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
            corresponds to `sequence_length`.
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NOptional[tf.Tensor]r   List[tf.Tensor] | Nonero   Tuple[tf.Tensor, ...] | Noner   r   )	r   r   r   r   r   __annotations__ro   r   r   r'   r'   r'   r<   r   F  s   
 r   c                   @  N   e Zd ZU dZdZded< dZded< dZded< dZd	ed
< dZ	d	ed< dS )TFXLNetLMHeadModelOutputa  
    Output type of [`TFXLNetLMHeadModel`].

    Args:
        loss (`tf.Tensor` of shape *(1,)*, *optional*, returned when `labels` is provided)
            Language modeling loss (for next-token prediction).
        logits (`tf.Tensor` of shape `(batch_size, num_predict, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

            `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
            corresponds to `sequence_length`.
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Ntf.Tensor | Nonelossr   logitsr   ro   r   r   r   
r   r   r   r   r  r   r  ro   r   r   r'   r'   r'   r<   r   h  s   
 r   c                   @  r   )&TFXLNetForSequenceClassificationOutputa)  
    Output type of [`TFXLNetForSequenceClassification`].

    Args:
        loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `label` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr  r  r   r  r   ro   r   r   r   r  r'   r'   r'   r<   r       
 r  c                   @  r   )#TFXLNetForTokenClassificationOutputa  
    Output type of [`TFXLNetForTokenClassificationOutput`].

    Args:
        loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided) :
            Classification loss.
        logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.num_labels)`):
            Classification scores (before SoftMax).
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr  r  r   r  r   ro   r   r   r   r  r'   r'   r'   r<   r    r  r  c                   @  r   )TFXLNetForMultipleChoiceOutputa.  
    Output type of [`TFXLNetForMultipleChoice`].

    Args:
        loss (`tf.Tensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
            Classification loss.
        logits (`tf.Tensor` of shape `(batch_size, num_choices)`):
            *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

            Classification scores (before SoftMax).
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr  r  r   r  r   ro   r   r   r   r  r'   r'   r'   r<   r    s   
 r  c                   @  sZ   e Zd ZU dZdZded< dZded< dZded< dZded	< dZ	d
ed< dZ
d
ed< dS )'TFXLNetForQuestionAnsweringSimpleOutputa  
    Output type of [`TFXLNetForQuestionAnsweringSimple`].

    Args:
        loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
        start_logits (`tf.Tensor` of shape `(batch_size, sequence_length,)`):
            Span-start scores (before SoftMax).
        end_logits (`tf.Tensor` of shape `(batch_size, sequence_length,)`):
            Span-end scores (before SoftMax).
        mems (`List[tf.Tensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr  r  r   start_logits
end_logitsr   ro   r   r   r   )r   r   r   r   r  r   r
  r  ro   r   r   r'   r'   r'   r<   r	    s   
 r	  a{	  

    This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
    as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
    behavior.

    <Tip>

    TensorFlow models and layers in `transformers` accept two formats as input:

    - having all inputs as keyword arguments (like PyTorch models), or
    - having all inputs as a list, tuple or dict in the first positional argument.

    The reason the second format is supported is that Keras methods prefer this format when passing inputs to models
    and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just
    pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second
    format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with
    the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first
    positional argument:

    - a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
    - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
    `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`
    - a dictionary with one or several input Tensors associated to the input names given in the docstring:
    `model({"input_ids": input_ids, "token_type_ids": token_type_ids})`

    Note that when creating models and layers with
    [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry
    about any of this, as you can just pass inputs like you would to any other Python function!

    </Tip>

    Parameters:
        config ([`XLNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aP  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        input_mask (`torch.FloatTensor` of shape `{0}`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z_The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.c                      sn   e Zd Z fddZeeedee	e
ed														dd ddZd!ddZ  ZS )"TFXLNetModelc                   s,   t  j|g|R i | t|dd| _d S )Nr   r   )r(   r)   r   r   r8   r6   inputsr9   r:   r'   r<   r)     s   zTFXLNetModel.__init__batch_size, sequence_length
checkpointoutput_typer   NFr   r   r   rp   ro   r   rq   r   r   rg   r   r   rr   r0   r   r   r_   rs   return+Union[TFXLNetModelOutput, Tuple[tf.Tensor]]c                 C  s*   | j |||||||||	|
||||d}|S )Nr   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r_   )r   )r8   r   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r_   rz   r'   r'   r<   r{     s"   zTFXLNetModel.callc                 C  sd   | j rd S d| _ t| dd d ur0t| jj | jd  W d    d S 1 s)w   Y  d S d S )NTr   )rL   rM   rN   rO   r   r&   rP   r   r'   r'   r<   rP     s   "zTFXLNetModel.buildr   )r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r_   rs   r  r  rS   )r   r   r   r)   r   r   XLNET_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr{   rP   r   r'   r'   r:   r<   r    s2    $r  zt
    XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c                      s   e Zd Z fddZdd Zdd Zd&dd	Zeee	
d
eeed															d'd(d"d#Zd)d$d%Z  ZS )*TFXLNetLMHeadModelc                   sF   t  j|g|R i | t|dd| _t|| jjdd| _d| _d S )Nr   r   lm_lossF)r(   r)   r   r   r   r   r  Zsupports_xla_generationr  r:   r'   r<   r)     s   
zTFXLNetLMHeadModel.__init__c                 C  r   rS   )r  r   r'   r'   r<   get_lm_head  r   zTFXLNetLMHeadModel.get_lm_headc                 C  s   t dt | jd | jj S )NzMThe method get_prefix_bias_name is deprecated. Please use `get_bias` instead./)warningswarnFutureWarningr&   r  r   r'   r'   r<   get_prefix_bias_name  s   z'TFXLNetLMHeadModel.get_prefix_bias_nameNc                   s  |j d }tj|df|jd}d |r&tj|d d   d f |gdd}n	tj||gdd}|j d }t|||d f}	t||df}
tj|	|
gdd}	t|d|d f}t|ddf}tj||gdd}||	||d}|rt fdd	|D |d
< |S )Nr   r    r   rI   r]   rX   )r   r   rq   r   c                 3  s,    | ]}|d   d d d d f V  qd S rS   r'   )r   Z
layer_pastoffsetr'   r<   r     s   * zCTFXLNetLMHeadModel.prepare_inputs_for_generation.<locals>.<genexpr>ro   )r?   rN   rE   ra   ru   r   r   )r8   r  Zpast_key_valuesr   r9   Zeffective_batch_sizeZdummy_tokenr   Zsequence_lengthr   Zperm_mask_seq_endrq   Ztarget_mapping_seq_endr'   r"  r<   prepare_inputs_for_generation  s*   
&
z0TFXLNetLMHeadModel.prepare_inputs_for_generationr  )r  r   Fr   r   r   rp   ro   r   rq   r   r   rg   r   r   rr   r0   r   r   labelsr_   rs   r  1Union[TFXLNetLMHeadModelOutput, Tuple[tf.Tensor]]c                 C  s   | j |||||||||	|
||||d}|d }| j||d}d}|dur*| ||}|s@|f|dd  }|dur>|f| S |S t|||j|j|jdS )a  
        labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
            config.vocab_size - 1]`.

        Return:

        Examples:

        ```python
        >>> import tensorflow as tf
        >>> import numpy as np
        >>> from transformers import AutoTokenizer, TFXLNetLMHeadModel

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-large-cased")
        >>> model = TFXLNetLMHeadModel.from_pretrained("xlnet/xlnet-large-cased")

        >>> # We show how to setup inputs to predict a next token using a bi-directional context.
        >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=True))[
        ...     None, :
        ... ]  # We will predict the masked token

        >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1]))
        >>> perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token

        >>> target_mapping = np.zeros(
        ...     (1, 1, input_ids.shape[1])
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(
        ...     input_ids,
        ...     perm_mask=tf.constant(perm_mask, dtype=tf.float32),
        ...     target_mapping=tf.constant(target_mapping, dtype=tf.float32),
        ... )

        >>> next_token_logits = outputs[
        ...     0
        ... ]  # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
        ```r  r   r^   Nr    r  r  ro   r   r   )r   r  hf_compute_lossr   ro   r   r   )r8   r   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r%  r_   transformer_outputsZhidden_stater  r  rm   r'   r'   r<   r{     s>   ?zTFXLNetLMHeadModel.callc                 C  r   )NTr   r  )rL   rM   rN   rO   r   r&   rP   r  r   r'   r'   r<   rP   h  r   zTFXLNetLMHeadModel.build)NNNNNNNNNNNNNNNNF) r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r%  rp   r_   rs   r  r&  rS   )r   r   r   r)   r  r!  r$  r   r   r  r  r   r   r  r{   rP   r   r'   r'   r:   r<   r    s2    
'
_r  z
    XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                      p   e Zd Z fddZeeedee	e
ed															d d!ddZd"ddZ  ZS )# TFXLNetForSequenceClassificationc                   sh   t  j|g|R i | |j| _t|dd| _t||jdd| _tj	j
|jt|jdd| _|| _d S )Nr   r   sequence_summaryr   logits_projr   )r(   r)   
num_labelsr   r   r   r/   r-  r   r1   r   r   r.  r6   r  r:   r'   r<   r)   |  s   
z)TFXLNetForSequenceClassification.__init__r  r  NFr   r   r   rp   ro   r   rq   r   r   rg   r   r   rr   r0   r   r   r%  r_   rs   r  ?Union[TFXLNetForSequenceClassificationOutput, Tuple[tf.Tensor]]c                 C  s   | j |||||||||	|
||||d}|d }| |}| |}|du r'dn| ||}|sC|f|dd  }|durA|f| S |S t|||j|j|jdS )a  
        labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r  r   Nr    r'  )r   r-  r.  r(  r  ro   r   r   r8   r   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r%  r_   r)  rm   r  r  r'   r'   r<   r{     s<   

z%TFXLNetForSequenceClassification.callc                 C    | j rd S d| _ t| dd d ur-t| jj | jd  W d    n1 s(w   Y  t| dd d urRt| jj | jd  W d    n1 sMw   Y  t| dd d urt| jj | jd d | j	j
g W d    d S 1 sxw   Y  d S d S NTr   r-  r.  rL   rM   rN   rO   r   r&   rP   r-  r.  r6   r*   r   r'   r'   r<   rP         "z&TFXLNetForSequenceClassification.buildr*  ) r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r%  rp   r_   rs   r  r0  rS   )r   r   r   r)   r   r   r  r  r   r  r  r  r{   rP   r   r'   r'   r:   r<   r,  t  s4    ;r,  z
    XLNET Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                      r+  )#TFXLNetForMultipleChoicec                   s^   t  j|g|R i | t|dd| _t||jdd| _tjj	dt
|jdd| _|| _d S )Nr   r   r-  r   r    r.  r   )r(   r)   r   r   r   r/   r-  r   r1   r   r   r.  r6   r  r:   r'   r<   r)     s   
z!TFXLNetForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr  NFr   r   r   rp   r   r   ro   r   rq   rg   r   r   rr   r0   r   r   r%  r_   rs   r  7Union[TFXLNetForMultipleChoiceOutput, Tuple[tf.Tensor]]c                 C  s~  |durt |d }t |d }nt |	d }t |	d }|dur)t|d|fnd}|dur7t|d|fnd}|durEt|d|fnd}|durSt|d|fnd}|	durft|	d|t |	d fnd}| j||||||||||
||||d}|d }| |}| |}t|d|f}|du rdn| ||}|s|f|dd  }|dur|f| S |S t|||j|j	|j
dS )	a5  
        labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
        Nr    rI   rX   r   )r   r_   r   r'  )r   rN   rY   r   r-  r.  r(  r  ro   r   r   )r8   r   r   r   r   ro   r   rq   rg   r   r   r0   r   r   r%  r_   Znum_choicesZ
seq_lengthZflat_input_idsZflat_attention_maskZflat_token_type_idsZflat_input_maskZflat_inputs_embedsr)  rm   r  Zreshaped_logitsr  r'   r'   r<   r{     sX   

zTFXLNetForMultipleChoice.callc                 C  r2  r3  r4  r   r'   r'   r<   rP   ?  r5  zTFXLNetForMultipleChoice.buildr*  ) r   r   r   rp   r   rp   r   rp   ro   rp   r   rp   rq   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r%  rp   r_   rs   r  r7  rS   )r   r   r   r)   r   r   r  r  r   r  r  r  r{   rP   r   r'   r'   r:   r<   r6    s4    Jr6  z
    XLNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                      r+  )#TFXLNetForTokenClassificationc                   sV   t  j|g|R i | |j| _t|dd| _tjj|jt|j	dd| _
|| _d S )Nr   r   
classifierr   )r(   r)   r/  r   r   r   r1   r   r   r/   r9  r6   r  r:   r'   r<   r)   V  s   
z&TFXLNetForTokenClassification.__init__r  r  NFr   r   r   rp   ro   r   rq   r   r   rg   r   r   rr   r0   r   r   r%  r_   rs   r  <Union[TFXLNetForTokenClassificationOutput, Tuple[tf.Tensor]]c                 C  s   | j |||||||||	|
||||d}|d }| |}|du r"dn| ||}|s>|f|dd  }|dur<|f| S |S t|||j|j|jdS )z
        labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r  r   Nr    r'  )r   r9  r(  r  ro   r   r   r1  r'   r'   r<   r{   `  s:   
z"TFXLNetForTokenClassification.callc                 C     | j rd S d| _ t| dd d ur-t| jj | jd  W d    n1 s(w   Y  t| dd d urZt| jj | jd d | jj	g W d    d S 1 sSw   Y  d S d S )NTr   r9  )
rL   rM   rN   rO   r   r&   rP   r9  r6   hidden_sizer   r'   r'   r<   rP        "z#TFXLNetForTokenClassification.buildr*  ) r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r%  rp   r_   rs   r  r:  rS   )r   r   r   r)   r   r   r  r  r   r  r  r  r{   rP   r   r'   r'   r:   r<   r8  N  s4    
7r8  z
    XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                      sr   e Zd Z fddZeeedee	e
ed																d!d"ddZd#dd Z  ZS )$!TFXLNetForQuestionAnsweringSimplec                   sN   t  j|g|R i | t|dd| _tjj|jt|j	dd| _
|| _d S )Nr   r   
qa_outputsr   )r(   r)   r   r   r   r1   r   r/  r   r/   r?  r6   r  r:   r'   r<   r)     s   
z*TFXLNetForQuestionAnsweringSimple.__init__r  r  NFr   r   r   rp   ro   r   rq   r   r   rg   r   r   rr   r0   r   r   start_positionsend_positionsr_   rs   r  @Union[TFXLNetForQuestionAnsweringSimpleOutput, Tuple[tf.Tensor]]c                 C  s   | j |||||||||	|
||||d}|d }| |}tj|ddd\}}tj|dd}tj|dd}d}|durN|durNd|i}||d< | |||f}|se||f|d	d  }|durc|f| S |S t||||j|j|j	d
S )a  
        start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        r  r   rI   rX   r]   NZstart_positionZend_positionr    )r  r
  r  ro   r   r   )
r   r?  rN   splitZsqueezer(  r	  ro   r   r   )r8   r   r   ro   r   rq   r   r   rg   r   r   r0   r   r   r@  rA  r_   r)  Zsequence_outputr  r
  r  r  r%  rm   r'   r'   r<   r{     sJ   $
z&TFXLNetForQuestionAnsweringSimple.callc                 C  r;  )NTr   r?  )
rL   rM   rN   rO   r   r&   rP   r?  r6   r<  r   r'   r'   r<   rP     r=  z'TFXLNetForQuestionAnsweringSimple.build)NNNNNNNNNNNNNNNF)"r   r   r   rp   ro   rp   r   rp   rq   rp   r   rp   r   rp   rg   rp   r   rp   r   rr   r0   rr   r   rr   r   rr   r@  rp   rA  rp   r_   rs   r  rB  rS   )r   r   r   r)   r   r   r  r  r   r  r	  r  r{   rP   r   r'   r'   r:   r<   r>    s6    Gr>  )r6  r>  r,  r8  r  r   r  r   )Hr   
__future__r   r  dataclassesr   typingr   r   r   r   numpynpZ
tensorflowrN   Zactivations_tfr	   Zmodeling_tf_utilsr
   r   r   r   r   r   r   r   r   r   r   r   r   Ztf_utilsr   r   r   utilsr   r   r   r   r   r   Zconfiguration_xlnetr!   Z
get_loggerr   loggerr  r  r1   ZLayerr"   r   r   r   r   r   r   r   r  r  r  r	  ZXLNET_START_DOCSTRINGr  r  r  r,  r6  r8  r>  __all__r'   r'   r'   r<   <module>   s   < 
  *7   o
!$!!#$*I9 %_mUc