
    fTh                       S r SSKrSSKJr  SSKJrJrJrJrJ	r	  SSK
r
SSK
Jr  SSKJrJrJr  SSKJrJr  SS	KJr  SS
KJr  SSKJr  SSKJrJrJr  SSKJr  \R>                  " \ 5      r!SBS jr"S r# " S S\RH                  5      r% " S S\RH                  5      r& " S S\RH                  5      r' " S S\RH                  5      r( " S S\RH                  5      r) " S S\RH                  5      r* " S S\RH                  5      r+\ " S S \5      5       r,\ " S! S"\5      5       r-\ " S# S$\5      5       r.\ " S% S&\5      5       r/\ " S' S(\5      5       r0\ " S) S*\5      5       r1\ " S+ S,\5      5       r2\ " S- S.\5      5       r3\ " S/ S0\,5      5       r4\" S1S29 " S3 S4\,\5      5       r5\" S5S29 " S6 S7\,5      5       r6\ " S8 S9\,5      5       r7\ " S: S;\,5      5       r8\" S<S29 " S= S>\,5      5       r9\ " S? S@\,5      5       r:/ SAQr;g)Cz
PyTorch XLNet model.
    N)	dataclass)CallableListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FNget_activation)GenerationMixin)PreTrainedModel)apply_chunking_to_forward)ModelOutputauto_docstringlogging   )XLNetConfigc                 f   0 n[        U S5      (       Ga  [        U S5      (       a  U R                  R                  US'   [        U S5      (       aL  SU;   aF  U R                  R                  R
                  US'   U R                  R                  R                  US'   [        U S5      (       ao  UR                  bb  SUR                   S	3U;   aN  U R                  R
                  USUR                   S	3'   U R                  R                  USUR                   S
3'   U R                  n UR                  U R                  R
                  U R                  S.5        [        U R                  5       GH  u  pESU S3nUR                  US-   UR                  R                  R
                  US-   UR                  R                  R                  US-   UR                  R                   US-   UR                  R"                  US-   UR                  R$                  US-   UR                  R&                  US-   UR                  R(                  US-   UR*                  R                  R
                  US-   UR*                  R                  R                  US-   UR*                  R,                  R
                  US-   UR*                  R,                  R                  US-   UR*                  R.                  R
                  US-   UR*                  R.                  R                  05        GM     UR0                  (       a  / n/ n/ n	/ n
U R                   H  nUR3                  UR                  R4                  5        UR3                  UR                  R6                  5        U	R3                  UR                  R8                  5        U
R3                  UR                  R:                  5        M     O4U R4                  /nU R6                  /nU R8                  /n	U R:                  /n
UR                  UUU	U
S.5        U$ )z
A map of modules from TF to PyTorch. I use a map to keep the PyTorch model as identical to the original PyTorch
model as possible.
transformerlm_losszmodel/lm_loss/biassequence_summaryz%model/sequnece_summary/summary/kernelz#model/sequnece_summary/summary/biaslogits_projzmodel/regression_z/logit/kernelz/logit/bias)z-model/transformer/word_embedding/lookup_tablez#model/transformer/mask_emb/mask_embzmodel/transformer/layer_/zrel_attn/LayerNorm/gammazrel_attn/LayerNorm/betazrel_attn/o/kernelzrel_attn/q/kernelzrel_attn/k/kernelzrel_attn/r/kernelzrel_attn/v/kernelzff/LayerNorm/gammazff/LayerNorm/betazff/layer_1/kernelzff/layer_1/biaszff/layer_2/kernelzff/layer_2/bias)zmodel/transformer/r_r_biaszmodel/transformer/r_w_biaszmodel/transformer/r_s_biaszmodel/transformer/seg_embed)hasattrr   biasr   summaryweightfinetuning_taskr   r   updateword_embeddingmask_emb	enumeratelayerrel_attn
layer_normoqkrvfflayer_1layer_2untie_rappendr_r_biasr_w_biasr_s_bias	seg_embed)modelconfig
tf_weightstf_to_pt_mapib	layer_strr_r_listr_w_listr_s_listseg_embed_lists              `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/xlnet/modeling_xlnet.pybuild_tf_xlnet_to_pytorch_maprD   '   s    Lum$$5)$$161C1CL-.5,--2Y]g2gDIDZDZDbDbDiDiL@ABGBXBXB`B`BeBeL>?E=))&&2#F$:$:#;=IZWV[VgVgVnVnL,V-C-C,DMRSTYTeTeTjTjL,V-C-C,DKPQ !! =B=Q=Q=X=X38>>	
 %++&.qc3	66

8M8M8T8T55qzz7L7L7Q7Q//////////00!$$//2H2H//1E1E//1D1D--qtt||/@/@//1D1D--qtt||/@/@	
 '* ~~AOOAJJ//0OOAJJ//0OOAJJ//0!!!**"6"67	  NN#NN#NN#//**2*2*2+9		
     c                     SSK nSSKnUR
                  R                  U5      n0 nU H?  u  px[        R                  SU SU 35        UR
                  R                  X'5      n	XU'   MA     [        XU5      n
U
R                  5        GH  u  p{[        R                  SU 35        Xv;  a  [        R                  U S35        M=  Xg   n	SU;   a8  S	U;   d  S
U;   d  SU;   a&  [        R                  S5        UR                  U	5      n	[        U[        5      (       a  [        U5      U	R                  S   :X  d$   S[        U5       SU	R                  S    S35       e[!        U5       H  u  pXS4   n UR                  UR                  :X  d"   SUR                   SUR                   S35       e [        R                  SU SU 35        [&        R(                  " U5      Ul        M     Oq UR                  U	R                  :X  d"   SUR                   SU	R                   S35       e [        R                  SU 35        [&        R(                  " U	5      Ul        UR-                  US5        UR-                  US-   S5        UR-                  US-   S5        GM     [        R                  SSR/                  UR1                  5       5       35        U $ ! [         a    [        R	                  S5        e f = f! ["         a1  nU=R$                  UR                  UR                  4-  sl        e SnAff = f! ["         a1  nU=R$                  UR                  U	R                  4-  sl        e SnAff = f)z&Load tf checkpoints in a pytorch modelr   NzLoading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.zLoading TF weight z with shape z
Importing z( not in tf pre-trained weights, skippingkernelr/   r    logitTransposingzPointer length z and array length z mismatched.zPointer shape z and array shape zInitialize PyTorch weight z for layer z/Adamz/Adam_1z%Weights not copied to PyTorch model: z, )numpy
tensorflowImportErrorloggererrortrainlist_variablesinfoload_variablerD   items	transpose
isinstancelistlenshaper&   AssertionErrorargstorch
from_numpydatapopjoinkeys)r8   r9   tf_pathnptf	init_varsr:   namerX   arrayr;   pointerr<   p_iarr_ies                   rC   load_tf_weights_in_xlnetrk   z   s%    ''0IJ (l5'BC&&w5 4 ! 1
KL%++-j'(!KK4& HIJ  td1BgQUoKK&LL'Egt$$w<5;;q>1 !#g,/A%++a.AQQ\]1 $G,f993 (3DU[[MQ\]3 8k!MN ++E2 -}}3 $W]]O3DU[[MQ\]3 KK4TF;< ++E2GLtT"tg~t,ti'.O .R KK7		*//BS8T7UVWLw  Q	
 	N & FFsyy%++66F " 7==%++66s;   K 3<K8,<L6!K58
L3,L..L36
M1 ,M,,M1c                      ^  \ rS rSrU 4S jrS r\S
S j5       r\S
S j5       r    SS jr	SS jr
    SS jrS	rU =r$ )XLNetRelativeAttention   c                   > [         TU ]  5         UR                  UR                  -  S:w  a%  [	        SUR                   SUR                   35      eUR                  U l        UR
                  U l        UR                  U l        SUR
                  S-  -  U l        [        R                  " [        R                  " UR                  U R                  U R
                  5      5      U l        [        R                  " [        R                  " UR                  U R                  U R
                  5      5      U l        [        R                  " [        R                  " UR                  U R                  U R
                  5      5      U l        [        R                  " [        R                  " UR                  U R                  U R
                  5      5      U l        [        R                  " [        R                  " UR                  U R                  U R
                  5      5      U l        [        R                  " [        R                  " U R                  U R
                  5      5      U l        [        R                  " [        R                  " U R                  U R
                  5      5      U l        [        R                  " [        R                  " U R                  U R
                  5      5      U l        [        R                  " [        R                  " SU R                  U R
                  5      5      U l        [        R(                  " UR                  UR*                  S9U l        [        R.                  " UR0                  5      U l        g )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (r         ?   eps)super__init__d_modeln_head
ValueErrord_headscaler	   	Parameterr[   FloatTensorr+   r,   r.   r*   r-   r4   r6   r5   r7   	LayerNormlayer_norm_epsr)   Dropoutdropoutselfr9   	__class__s     rC   ru   XLNetRelativeAttention.__init__   s$   >>FMM)Q.#FNN#3 4 --* 
 mmmm~~&--,-
e//T[[YZe//T[[YZe//T[[YZe//T[[YZe//T[[YZU%6%6t{{DKK%PQU%6%6t{{DKK%PQU%6%6t{{DKK%PQe&7&74;;&TU,,v~~6;P;PQzz&..1rE   c                     [         eNNotImplementedError)r   headss     rC   prune_heads"XLNetRelativeAttention.prune_heads       !!rE   c           
      <   U R                   nU R                  US   US   US   US   5      n U SS2S4   n U R                  US   US   S-
  US   US   5      n [        R                  " U S[        R                  " XR
                  [        R                  S95      n U $ )z<perform relative shift to form the relative attention score.r   r   rq   r   N.devicedtyperX   reshaper[   index_selectaranger   longxklenx_sizes      rC   	rel_shift XLNetRelativeAttention.rel_shift   s     IIfQiF1IvayAab#gJIIfQiQq	6!9Eq!U\\$xxuzz%Z[rE   c           
      L   U R                   nU R                  US   US   US   US   5      n U S S 2S S 2SS 2S S 24   n U R                  US   US   US   US   S-
  5      n [        R                  " U S[        R                  " XR
                  [        R                  S95      n U $ )Nr   r   r   rq   r   r   r   s      rC   rel_shift_bnij%XLNetRelativeAttention.rel_shift_bnij   s    IIfQiF1IvayAaABkNIIfQiF1Ivay1}E q!U\\$xxuzz%Z[ rE   c	                 J   [         R                  " SXR                  -   U5      n	[         R                  " SXR                  -   U5      n
U R	                  XR
                  S   S9n
Uc  SnOE[         R                  " SXR                  -   U R                  5      n[         R                  " SX[5      nX-   U-   U R                  -  nUbY  UR                  [         R                  :X  a  US[         R                  " SU5      -  -
  nOUS	[         R                  " SU5      -  -
  n[        R                  R                  USS
9nU R                  U5      nUb  U[         R                  " SU5      -  n[         R                  " SX5      nU(       a  U[         R                  " SU5      4$ U$ )z.Core relative positional attention operations.zibnd,jbnd->bnijr   )r   r   zibnd,snd->ibnszijbs,ibns->bnij  z
ijbn->bnijꌠ9Y>)Fdimzbnij,jbnd->ibndz
bnij->ijbn)r[   einsumr5   r4   r   rX   r6   r7   rz   r   float16r	   
functionalsoftmaxr   )r   q_headk_head_hv_head_hk_head_rseg_mat	attn_mask	head_maskoutput_attentionsacbdef
attn_score	attn_probattn_vecs                  rC   rel_attn_core$XLNetRelativeAttention.rel_attn_core   sg    \\+Vmm-CXN \\+Vmm-CXN  ((1+ 6 ?B.0FWB/=B gldjj0
 %--/'%%,,|Y2W*WW
'$lI1V*VV
 MM))*!)<	LL+	  !ELLy$III << 19GU\\,	BBBrE   c                     [         R                  " SX R                  5      nU R                  U5      nU(       a  XA-   nU R	                  U5      nU$ )zPost-attention processing.zibnd,hnd->ibh)r[   r   r*   r   r)   )r   hr   residualattn_outoutputs         rC   post_attention%XLNetRelativeAttention.post_attention-  sC     <<66B<<)|H*rE   c                    UGb  Ub+  UR                  5       S:  a  [        R                  " Xq/SS9nOUn[        R                  " SXR                  5      n[        R                  " SXR
                  5      n[        R                  " SXPR                  5      n[        R                  " SXR                  5      nU R                  UUUUUUU	U
S9nU
(       a  Uu  nnU R                  UU5      n[        R                  " SX R                  5      nUbS  [        R                  " SUU5      nU R                  UUUUUUU	U
S9nU
(       a  Uu  nn[        R                  " SUU5      nO"U R                  UUUUUUU	U
S9nU
(       a  Uu  nnU R                  UU5      nU
(       a  WW4nGOUb+  UR                  5       S:  a  [        R                  " Xq/SS9nOUn[        R                  " SXR                  5      n[        R                  " SXR                  5      n[        R                  " SXR
                  5      n[        R                  " SUR                  U R                  R                  5      U R                  5      nU R                  UUUUUUU	U
S9nU
(       a  Uu  nnU R                  UU5      nS nUU4nU
(       a  UW4-   nU$ )Nr   r   r   zibh,hnd->ibnd)r   r   r   r   zmbnd,mlb->lbndzlbnd,mlb->mbnd)r   r[   catr   r,   r.   r-   r+   r   r   typer   )r   r   gattn_mask_hattn_mask_gr-   r   memstarget_mappingr   r   r   r   r   r   q_head_h
attn_vec_hattn_prob_houtput_hq_head_g
attn_vec_gattn_prob_goutput_gr   r   outputss                             rC   forwardXLNetRelativeAttention.forward9  s    = DHHJNii	q1 ||OS&&AH ||OS&&AH ||OQ?H ||OQ?H ++%#"3 , 	J !*4'
K **1j9H ||OQ?H ) <<(8(NS!//#)'&7 0 	
 %.8+J"\\*:JW
!//#)'&7 0 	
 %.8+J **1j9H '4	 DHHJNii	q1 ||OQ?H||OS&&AH||OS&&AH ||OQVVDFFLL5I466RH ))%#"3 * 	H !&.#) **1h7HHX&,GrE   )ry   rv   r   r,   r)   rw   r*   r+   r-   r4   r6   r5   rz   r7   r.   NNNF)T)__name__
__module____qualname____firstlineno__ru   r   staticmethodr   r   r   r   r   __static_attributes____classcell__r   s   @rC   rm   rm      sh    28" 
 
  ( 2h
( E ErE   rm   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )XLNetFeedForwardi  c                   > [         TU ]  5         [        R                  " UR                  UR
                  S9U l        [        R                  " UR                  UR                  5      U l	        [        R                  " UR                  UR                  5      U l
        [        R                  " UR                  5      U l        [        UR                  [        5      (       a  [         UR                     U l        g UR                  U l        g )Nrr   )rt   ru   r	   r}   rv   r~   r)   Lineard_innerr0   r1   r   r   rU   ff_activationstrr   activation_functionr   s     rC   ru   XLNetFeedForward.__init__  s    ,,v~~6;P;PQyy@yy@zz&..1f**C00'-f.B.B'CD$'-';';D$rE   c                     UnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  X!-   5      nU$ r   )r0   r   r   r1   r)   )r   inpr   s      rC   r   XLNetFeedForward.forward  sd    f%))&1f%f%f%.rE   )r   r   r0   r1   r)   )r   r   r   r   ru   r   r   r   r   s   @rC   r   r     s    	< rE   r   c                   @   ^  \ rS rSrU 4S jr    SS jrS rSrU =r$ )
XLNetLayeri  c                    > [         TU ]  5         [        U5      U l        [	        U5      U l        [        R                  " UR                  5      U l        UR                  U l	        SU l
        g Nr   )rt   ru   rm   r(   r   r/   r	   r   r   chunk_size_feed_forwardseq_len_dimr   s     rC   ru   XLNetLayer.__init__  sP    .v6"6*zz&..1'-'E'E$rE   c                    U R                  UUUUUUUUU	U
S9
nUS S u  pUb,  [        U R                  U R                  U R                  U5      n[        U R                  U R                  U R                  U5      nX4USS  -   nU$ )N)r   r   r   r   rq   )r(   r   ff_chunkr   r   )r   r   r   r   r   r-   r   r   r   r   r   r   s               rC   r   XLNetLayer.forward  s     --)/   
 %Ra[0t;;T=M=MxH -T]]D<X<XZ^ZjZjltu&4rE   c                 (    U R                  U5      nU$ r   )r/   )r   output_xs     rC   r   XLNetLayer.ff_chunk  s    778$rE   )r   r   r/   r(   r   r   )	r   r   r   r   ru   r   r   r   r   r   s   @rC   r   r     s&      "H rE   r   c                      ^  \ rS rSrSrS\4U 4S jjr S
S\R                  S\	\R                     S\R                  4S jjr
S	rU =r$ )XLNetPoolerStartLogitsi  z
Compute SQuAD start logits from sequence hidden states.

Args:
    config ([`XLNetConfig`]):
        The config used by the model, will be used to grab the `hidden_size` of the model.
r9   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g r   )rt   ru   r	   r   hidden_sizedenser   s     rC   ru   XLNetPoolerStartLogits.__init__  s&    YYv1115
rE   hidden_statesp_maskreturnc                     U R                  U5      R                  S5      nUb<  UR                  [        R                  :X  a  USU-
  -  SU-  -
  nU$ USU-
  -  SU-  -
  nU$ )a  
Args:
    hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
        The final hidden states of the model.
    p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
        Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
        should be masked.

Returns:
    `torch.FloatTensor`: The start logits for SQuAD.
r   r   r   r   )r   squeezer   r[   r   )r   r   r   r   s       rC   r   XLNetPoolerStartLogits.forward  so     JJ}%--b1||u}},V$uv~5  V$tf}4rE   )r   r   )r   r   r   r   __doc__r   ru   r[   r|   r   r   r   r   r   s   @rC   r   r     sT    6{ 6
 W["..8@ARAR8S			 rE   r   c                      ^  \ rS rSrSrS\4U 4S jjr   SS\R                  S\	\R                     S\	\R                     S\	\R                     S	\R                  4
S
 jjrSrU =r$ )XLNetPoolerEndLogitsi2  z
Compute SQuAD end logits from sequence hidden states.

Args:
    config ([`XLNetConfig`]):
        The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
        to use.
r9   c                 d  > [         TU ]  5         [        R                  " UR                  S-  UR                  5      U l        [        R                  " 5       U l        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  S5      U l
        g )Nrq   rr   r   )rt   ru   r	   r   r   dense_0Tanh
activationr}   r~   dense_1r   s     rC   ru   XLNetPoolerEndLogits.__init__<  st    yy!3!3a!79K9KL'')f&8&8f>S>STyy!3!3Q7rE   r   start_statesstart_positionsr   r   c                    Uc
  Uc   S5       eUbQ  UR                   SS u  pVUSS2SS4   R                  SSU5      nUR                  SU5      nUR                  SUS5      nU R                  [        R
                  " X/SS95      nU R                  U5      nU R                  U5      nU R                  U5      R                  S5      nUb<  UR                  [        R                  :X  a  USU-
  -  SU-  -
  nU$ USU-
  -  SU-  -
  nU$ )	ac  
Args:
    hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
        The final hidden states of the model.
    start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
        The hidden states of the first tokens for the labeled span.
    start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
        The position of the first token for the labeled span.
    p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
        Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
        should be masked.

<Tip>

One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
`start_states`.

</Tip>

Returns:
    `torch.FloatTensor`: The end logits for SQuAD.
N7One of start_states, start_positions should be not Noner   r   r   r   r   )rX   expandgatherr  r[   r   r  r}   r	  r   r   r   )r   r   r  r  r   slenhszr   s           rC   r   XLNetPoolerEndLogits.forwardC  s"   : '?+F 	
E	
F &%++BC0ID-atm<CCBCPO(//ODL'..r4<LLLM#@bIJOOANN1LLO##B'||u}},V$uv~5  V$tf}4rE   )r}   r  r  r	  NNNr   r   r   r   r  r   ru   r[   r|   r   
LongTensorr   r   r   r   s   @rC   r  r  2  s    8{ 8 596:.21((1 u0011 "%"2"23	1
 **+1 
		1 1rE   r  c                      ^  \ rS rSrSrS\4U 4S jjr   SS\R                  S\	\R                     S\	\R                     S\	\R                     S	\R                  4
S
 jjrSrU =r$ )XLNetPoolerAnswerClassix  z
Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

Args:
    config ([`XLNetConfig`]):
        The config used by the model, will be used to grab the `hidden_size` of the model.
r9   c                   > [         TU ]  5         [        R                  " UR                  S-  UR                  5      U l        [        R                  " 5       U l        [        R                  " UR                  SSS9U l        g )Nrq   r   Fr   )	rt   ru   r	   r   r   r  r  r  r	  r   s     rC   ru   XLNetPoolerAnswerClass.__init__  sX    yy!3!3a!79K9KL'')yy!3!3QUCrE   r   r  r  	cls_indexr   c                     UR                   S   nUc
  Uc   S5       eUb<  USS2SS4   R                  SSU5      nUR                  SU5      R                  S5      nUb=  USS2SS4   R                  SSU5      nUR                  SU5      R                  S5      nOUSS2SSS24   nU R	                  [
        R                  " X&/SS95      nU R                  U5      nU R                  U5      R                  S5      nU$ )a8  
Args:
    hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
        The final hidden states of the model.
    start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
        The hidden states of the first tokens for the labeled span.
    start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
        The position of the first token for the labeled span.
    cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
        Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

<Tip>

One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
`start_states`.

</Tip>

Returns:
    `torch.FloatTensor`: The SQuAD 2.0 answer class.
r   Nr  r  r   )	rX   r  r  r   r  r[   r   r  r	  )r   r   r  r  r  r  cls_token_stater   s           rC   r   XLNetPoolerAnswerClass.forward  s   : !!"%'?+F 	
E	
F &-atm<CCBCPO(//ODLLRPL !!T4-077BDI+222yAII"MO+Ar1H5OLLL#BKLOOALLO##B'rE   )r  r  r	  r  r  r   s   @rC   r  r  x  s    D{ D 596:04/((/ u001/ "%"2"23	/
 E,,-/ 
		/ /rE   r  c                      ^  \ rS rSrSrS\4U 4S jjr S
S\R                  S\	\R                     S\R                  4S jjrS	rU =r$ )XLNetSequenceSummaryi  a  
Compute a single vector summary of a sequence hidden states.

Args:
    config ([`XLNetConfig`]):
        The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
        config class of your model for the default values it uses):

        - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

            - `"last"` -- Take the last token hidden state (like XLNet)
            - `"first"` -- Take the first token hidden state (like Bert)
            - `"mean"` -- Take the mean of all tokens hidden states
            - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
            - `"attn"` -- Not implemented now, use multi-head attention

        - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
        - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
          (otherwise to `config.hidden_size`).
        - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
          another string or `None` will add no activation.
        - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
        - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
r9   c                   > [         TU ]  5         [        USS5      U l        U R                  S:X  a  [        e[
        R                  " 5       U l        [        US5      (       a  UR                  (       aq  [        US5      (       a.  UR                  (       a  UR                  S:  a  UR                  nOUR                  n[
        R                  " UR                  U5      U l        [        USS 5      nU(       a  [        U5      O[
        R                  " 5       U l        [
        R                  " 5       U l        [        US5      (       a5  UR"                  S:  a%  [
        R$                  " UR"                  5      U l        [
        R                  " 5       U l        [        US	5      (       a7  UR(                  S:  a&  [
        R$                  " UR(                  5      U l        g g g )
Nsummary_typelastattnsummary_use_projsummary_proj_to_labelsr   summary_activationsummary_first_dropoutsummary_last_dropout)rt   ru   getattrr$  r   r	   Identityr    r   r'  r(  
num_labelsr   r   r   r  first_dropoutr*  r   last_dropoutr+  )r   r9   num_classesactivation_stringr   s       rC   ru   XLNetSequenceSummary.__init__  sa   #FNFC& &%{{}6-..63J3Jv788V=Z=Z_e_p_pst_t$//$0099V%7%7EDL#F,@$GIZN3D$E`b`k`k`m[[]62338T8TWX8X!#F,H,H!IDKKM6122v7R7RUV7V "

6+F+F GD 8W2rE   r   r  r   c                    U R                   S:X  a  USS2S4   nGOU R                   S:X  a  USS2S4   nGOU R                   S:X  a  UR                  SS9nOU R                   S	:X  a  Uc?  [        R                  " US
SS2SS24   UR                  S   S-
  [        R
                  S9nOXUR                  S5      R                  S5      nUR                  SUR                  5       S-
  -  UR                  S5      4-   5      nUR                  SU5      R                  S5      nOU R                   S:X  a  [        eU R                  W5      nU R                  U5      nU R                  U5      nU R!                  U5      nU$ )a#  
Compute a single vector summary of a sequence hidden states.

Args:
    hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
        The hidden states of the last layer.
    cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
        Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

Returns:
    `torch.FloatTensor`: The summary of the sequence hidden states.
r%  Nr   firstr   meanr   r   r  .r  r   r   r&  )r$  r6  r[   	full_likerX   r   	unsqueezer  r   sizer  r   r   r/  r    r  r0  )r   r   r  r   s       rC   r   XLNetSequenceSummary.forward  sn    &"1b5)F')"1a4(F&("''A'.F+- !OO!#rr1*-!''+a/**	 &//3==bA	%,,Uimmo6I-JmN`N`acNdMf-fg	"))"i8@@DF&(%%##F+f%(""6*rE   )r  r/  r0  r    r$  r   r  r   s   @rC   r"  r"    sV    2H{ H< Y])"..);CEDTDT;U)			) )rE   r"  c                   &    \ rS rSr\r\rSrS r	Sr
g)XLNetPreTrainedModeli  r   c           	         [        U[        R                  5      (       ak  UR                  R                  R                  SU R                  R                  S9  UR                  b%  UR                  R                  R                  5         gg[        U[        R                  5      (       ax  UR                  R                  R                  SU R                  R                  S9  UR                  b2  UR                  R                  UR                     R                  5         gg[        U[        R                  5      (       aJ  UR                  R                  R                  5         UR                  R                  R                  S5        g[        U[        5      (       a  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  4	 H1  nUR                  R                  SU R                  R                  S9  M3     g[        U[0        5      (       a9  UR2                  R                  R                  SU R                  R                  S9  gg)zInitialize the weights.g        )r6  stdN      ?)rU   r	   r   r!   r]   normal_r9   initializer_ranger   zero_	Embeddingpadding_idxr}   fill_rm   r+   r,   r.   r*   r-   r4   r6   r5   r7   
XLNetModelr%   )r   moduleparams      rC   _init_weights"XLNetPreTrainedModel._init_weights#  s   fbii(( MM&&CT[[5R5R&S{{&  &&( '--MM&&CT[[5R5R&S!!-""6#5#56<<> .--KK""$MM$$S) 677  
 

""1N1N"O
 
++OO  ((ct{{7T7T(U ,rE    N)r   r   r   r   r   config_classrk   load_tf_weightsbase_model_prefixrJ  r   rL  rE   rC   r=  r=    s    L.O%VrE   r=  c                       \ rS rSr% Sr\R                  \S'   Sr\	\
\R                        \S'   Sr\	\\R                  S4      \S'   Sr\	\\R                  S4      \S'   S	rg)
XLNetModelOutputiC  a  
Output type of [`XLNetModel`].

Args:
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
        Sequence of hidden-states at the last layer of the model.

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
last_hidden_stateNr   .r   
attentionsrL  )r   r   r   r   r  r[   r|   __annotations__r   r   r   r   r   rS  r   rL  rE   rC   rQ  rQ  C  sm    4 (((.2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   rQ  c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   S
rg)XLNetLMHeadModelOutputie  a  
Output type of [`XLNetLMHeadModel`].

Args:
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided)
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
Nlosslogitsr   .r   rS  rL  r   r   r   r   r  rW  r   r[   r|   rT  rX  r   r   r   r   rS  r   rL  rE   rC   rV  rV  e  s    8 )-D(5$$
%,*.FHU&&'..2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   rV  c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   S
rg)$XLNetForSequenceClassificationOutputi  a  
Output type of [`XLNetForSequenceClassification`].

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `label` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
NrW  rX  r   .r   rS  rL  rY  rL  rE   rC   r[  r[        2 )-D(5$$
%,*.FHU&&'..2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   r[  c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   S
rg)!XLNetForTokenClassificationOutputi  a  
Output type of [`XLNetForTokenClassificationOutput`].

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) :
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
        Classification scores (before SoftMax).
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
NrW  rX  r   .r   rS  rL  rY  rL  rE   rC   r^  r^    r\  rE   r^  c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   S
rg)XLNetForMultipleChoiceOutputi  a  
Output type of [`XLNetForMultipleChoice`].

Args:
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

        Classification scores (before SoftMax).
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
NrW  rX  r   .r   rS  rL  rY  rL  rE   rC   r`  r`    s    6 )-D(5$$
%,*.FHU&&'..2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   r`  c                   $   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                  S4      \	S	'   Sr\\\R                  S4      \	S
'   Srg)%XLNetForQuestionAnsweringSimpleOutputi  a  
Output type of [`XLNetForQuestionAnsweringSimple`].

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
    start_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-start scores (before SoftMax).
    end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-end scores (before SoftMax).
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
NrW  start_logits
end_logitsr   .r   rS  rL  )r   r   r   r   r  rW  r   r[   r|   rT  rc  rd  r   r   r   r   rS  r   rL  rE   rC   rb  rb    s    6 )-D(5$$
%,04L(5,,-4.2J**+2.2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   rb  c                      \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S	'   Sr\\\R                        \	S
'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S'   Srg)XLNetForQuestionAnsweringOutputi  a
  
Output type of [`XLNetForQuestionAnswering`].

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
        Classification loss as the sum of start token, end token (and is_impossible if provided) classification
        losses.
    start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top config.start_n_top start token possibilities (beam-search).
    start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top config.start_n_top start token possibilities (beam-search).
    end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
        (beam-search).
    end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
    cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the `is_impossible` label of the answers.
    mems (`List[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
NrW  start_top_log_probsstart_top_indexend_top_log_probsend_top_index
cls_logitsr   .r   rS  rL  )r   r   r   r   r  rW  r   r[   r|   rT  rg  rh  r  ri  rj  rk  r   r   r   r   rS  r   rL  rE   rC   rf  rf    s    !F )-D(5$$
%,7;%"3"34;26OXe../659x 1 12904M8E,,-4.2J**+2.2D(4))*
+2=AM8E%"3"3S"89:A:>Ju00#567>rE   rf  c                      ^  \ rS rSrU 4S jrS rS rS rS rS r	\
SS j5       rSS	 jr\             SS
\\R                      S\\R                      S\\R                      S\\R                      S\\R                      S\\R                      S\\R                      S\\R                      S\\R                      S\\   S\\   S\\   S\\   S\\\4   4S jj5       rSrU =r$ )rG  iG  c                   > [         TU ]  U5        UR                  U l        UR                  U l        UR                  U l        UR
                  U l        UR                  U l        UR                  U l        UR                  U l        UR                  U l	        [        R                  " UR                  UR                  5      U l        [        R                  " [        R                   " SSUR                  5      5      U l        [        R$                  " ['        UR                  5       Vs/ s H  n[)        U5      PM     sn5      U l        [        R,                  " UR.                  5      U l        U R1                  5         g s  snf r   )rt   ru   mem_len	reuse_lenrv   same_length	attn_typebi_data	clamp_lenn_layerr	   rD  
vocab_sizer$   r{   r[   r|   r%   
ModuleListranger   r'   r   r   	post_init)r   r9   _r   s      rC   ru   XLNetModel.__init__I  s    ~~))~~!--))~~))~~ ll6+<+<fnnMU%6%6q!V^^%LM]]fnn@U#V@U1Jv$6@U#VW
zz&..1 		 $Ws   +Fc                     U R                   $ r   r$   r   s    rC   get_input_embeddingsXLNetModel.get_input_embeddings]  s    """rE   c                     Xl         g r   r|  r   new_embeddingss     rC   set_input_embeddingsXLNetModel.set_input_embeddings`  s    ,rE   c                     [         er   r   )r   heads_to_prunes     rC   _prune_headsXLNetModel._prune_headsc  r   rE   c                    [         R                  " XU-   4U R                  S9nU R                  (       aC  USS2SU24   R	                  S5      nUR                  US-   5        USS2SU24==   U-  ss'   U$ UR                  US-   5        U$ )a  
Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked.

Args:
    qlen: Sequence length
    mlen: Mask length

::

          same_length=False: same_length=True: <mlen > < qlen > <mlen > < qlen >
       ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
         [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
    qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
         [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
       v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]

)r   Nr   r   )r[   onesr   rp  triltriu_)r   qlenmlenmaskmask_los        rC   create_maskXLNetModel.create_maskf  s    $ zz4-dkkB1ete8n))"-GJJtax ETENg%N  JJtax rE   c                 4   U R                   b  U R                   S:  a  US U R                    nU R                  b  U R                  S:X  a  SnOU R                  * nUc  XS  nUR	                  5       $ [        R                  " X!/SS9US  nUR	                  5       $ )Nr   r   )ro  rn  r[   r   detach)r   curr_outprev_memcutoffnew_mems        rC   	cache_memXLNetModel.cache_mem  s    >>%$..1*< 0$..1H<<4<<1#4 F ll]Fw'G ~~ ii 4!<VWEG~~rE   c                     [         R                  " SX5      n[         R                  " [         R                  " U5      [         R                  " U5      /SS9nUS S 2S S S 24   nUb  UR                  SUS5      nU$ )Nzi,d->idr   r   )r[   r   r   sincosr  )pos_seqinv_freqbszsinusoid_inppos_embs        rC   positional_embeddingXLNetModel.positional_embedding  sf    ||IwA))UYY|4eii6MNTVW!T1*%?nnRb1GrE   c                    [         R                  " SU R                  S[         R                  S9R	                  5       nS[         R
                  " SX@R                  -  5      -  nU R                  S:X  a  X!* pvO-U R                  S:X  a  USpvO[        S	U R                   S
35      eU R                  (       Ga-  [         R                  " XgS[         R                  S9R	                  5       n[         R                  " U* U* S[         R                  S9R	                  5       n	U R                  S:  aN  UR                  U R                  * U R                  5      nU	R                  U R                  * U R                  5      n	Ub+  U R                  XUS-  5      n
U R                  XUS-  5      nO"U R                  X5      n
U R                  X5      n[         R                  " X/SS9nU$ [         R                  " XgS[         R                  S9R	                  5       nU R                  S:  a'  UR                  U R                  * U R                  5      nU R                  XU5      nU$ )Nr   g       @r7  r   i'  biunir   zUnknown `attn_type` .g      r@  rq   r   )r[   r   rv   int64floatpowrq  rx   rr  rs  clampr  r   )r   r  r   r  freq_seqr  begendfwd_pos_seqbwd_pos_seqfwd_pos_embbwd_pos_embr  s                rC   relative_positional_encoding'XLNetModel.relative_positional_encoding  s   <<4<<EKKHNNPuyyLL)@BB>>T!U^^u$R3DNN3C1EFF<<<,,sU[[IOOQK,,tcT3ekkJPPRK~~!)//P)//P"77sVWxX"77sVWxX"77N"77Nii :BG   ,,sU[[IOOQK~~!)//P//sKGrE   	input_idsattention_maskr   	perm_maskr   token_type_ids
input_maskr   inputs_embedsuse_memsr   output_hidden_statesreturn_dictr   c                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nSU;   a   [        R
                  " S[        5        US   n
U R                  (       a  U
b  U
OU R                   R                  n
OU
b  U
OU R                   R                  n
Ub  U	b  [        S5      eUb?  UR                  SS5      R                  5       nUR                  S   UR                  S   nnOMU	b?  U	R                  SS5      R                  5       n	U	R                  S   U	R                  S   nnO[        S5      eUb   UR                  SS5      R                  5       OSnUb   UR                  SS5      R                  5       OSnUb   UR                  SS5      R                  5       OSnUb!  UR                  SSS5      R                  5       OSnUb!  UR                  SSS5      R                  5       OSnUb  US   b  US   R                  S   OSnUU-   nU R                  nU R                   nU R"                  S	:X  a   U R%                  UU5      nUSS2SS2SS4   nO+U R"                  S
:X  a  SnO[        SU R"                   35      eUb
  Ub   S5       e Uc  Ub  SU-
  nUb  Ub	  US   U-   nOUb	  Uc  US   nOUc  Ub  UnOSnUbw  US:  aL  [&        R(                  " UR                  S   UU/5      R+                  U5      n[&        R,                  " UU/SS9nUc  USS2SS2SS2S4   nOUUSS2SS2SS2S4   -  nUb  US:  R+                  U5      nUb  [&        R.                  " U5      R+                  U5      * nUS:  a<  [&        R,                  " [&        R(                  " UU/5      R+                  U5      U/SS9nUUSS2SS2SS4   -   S:  R+                  U5      nOSnU	b  U	nOU R1                  U5      nU R3                  U5      nUb<  U R4                  R7                  UR                  S   US5      nU R3                  U5      nOSnUb  US:  a>  [&        R(                  " UU/[&        R8                  US9n[&        R,                  " UU/SS9nOUnUSS2S4   USSS24   :g  R9                  5       n[:        R<                  R?                  USS9R+                  U5      nOSnU RA                  UUUS9n U R+                  UR                   5      n U R3                  U 5      n Ub  URC                  5       S:X  a^  URE                  S5      RE                  S5      RE                  S5      RE                  S5      nUR7                  U RF                  SSSS5      nOCURC                  5       S:X  a/  URE                  S5      RE                  S5      RE                  S5      nUR+                  [I        U RK                  5       5      R                  S9nOS/U RF                  -  nSn!Uc  S/[M        U RN                  5      -  nU(       a  / OSn"U(       a  / OSn#[Q        U RN                  5       H  u  n$n%U
(       a  U!U RS                  UUU$   5      4-   n!U(       a  U#RU                  Ub  UU4OU5        U%" UUUUU UUU$   UUU$   US9
n&U&SS u  nnU(       d  Mk  U"RU                  U&S   5        M     U(       a  U#RU                  Ub  UU4OU5        U R3                  Ub  UOU5      n'U'R                  SSS5      R                  5       n'U
(       d  Sn!U(       a(  Ub  [W        S U# 5       5      n#O[W        S U# 5       5      n#U(       a(  Ub  [W        S U" 5       5      n"O[W        S U" 5       5      n"U(       d  [W        S U'U!U#U"4 5       5      $ [Y        U'U!U#U"S9$ )  
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.
N	use_cachezgThe `use_cache` argument is deprecated and will be removed in a future version, use `use_mems` instead.zDYou cannot specify both input_ids and inputs_embeds at the same timer   r   z5You have to specify either input_ids or inputs_embedsrq   r  r  zUnsupported attention type: z8You can only use one of input_mask (uses 1 for padding) r@  r   r   r   r   )r1  )r  r7  rL  )r   r   r-   r   r   r   r   r   c              3   t   #    U  H.  o  H%  o"R                  S SS5      R                  5       v   M'     M0     g7fr   r   rq   Npermute
contiguous).0hsr   s      rC   	<genexpr>%XLNetModel.forward.<locals>.<genexpr>  s5     %jP]"gibcii1a&8&C&C&E&Egi&EP]s   68c              3   b   #    U  H%  oR                  S SS5      R                  5       v   M'     g7fr  r  )r  r  s     rC   r  r    s)     %_Q^2jjAq&9&D&D&F&FQ^s   -/c              3   F   #    U  H  n[        S  U 5       5      v   M     g7f)c              3   d   #    U  H&  oR                  S SSS5      R                  5       v   M(     g7frq   r   r   r   Nr  )r  
att_streams     rC   r  /XLNetModel.forward.<locals>.<genexpr>.<genexpr>  s-     ZXY*,,Q1a8CCEEXY   .0N)tupler  ts     rC   r  r    s$      #dn_`EZXYZZZdns   !c              3   d   #    U  H&  oR                  S SSS5      R                  5       v   M(     g7fr  r  r  s     rC   r  r    s)     "Zz!99Q1a#8#C#C#E#Ezr  c              3   .   #    U  H  oc  M  Uv   M     g 7fr   rL  )r  r.   s     rC   r  r    s     c$Qq$Qs   	)rR  r   r   rS  )-r9   r   r  use_return_dictwarningswarnFutureWarningtraininguse_mems_trainuse_mems_evalrx   rT   r  rX   r  r   r   rq  r  r[   zerostor   eyer$   r   r%   r  r   r	   r   one_hotr  r   r9  rt  next
parametersrW   r'   r&   r  r3   r  rQ  )(r   r  r  r   r  r   r  r  r   r  r  r   r  r  kwargsr  r  r  r   dtype_floatr   r   	data_mask	mems_masknon_tgt_mask
word_emb_kr   
word_emb_qr   mem_padcat_idsr   r  new_memsrS  r   r<   layer_moduler   r   s(                                           rC   r   XLNetModel.forward  s   h 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]& MM
 k*H==#+#7xT[[=W=WH#+#7xT[[=V=VH
  ]%>cdd"!++Aq1<<>I!*IOOA,>#D#&)33Aq9DDFM%++A.0C0CA0F#D#TUUHVHb11!Q7BBDhl@J@VZ))!Q/::<\`
HVHb11!Q7BBDhl?H?TI%%aA.99;Z^	IWIc//1a8CCEim#'#3Q8KtAw}}QQRd{jj >>U"((t4I!!Qd"23I^^t#I;DNN;KLMM !^%;w=ww;g."<~-J!i&;"4(94I#	(9"4(II$9!II ax!KK);T3(GHKKIV	!IIy)&<!D	 %aAtm4	Yq!Q}55	 "Q**;7I !IIdO..y99Lax$yy%++tTl*C*F*Fy*QS_)`fhi&aD$6F)GG1LPPQZ[LL $&J,,Y7J<<
+%--n.B.B1.EsBOJ ||J/HH %ax++tSkFS))Wn$=1E( &ag.'$'2BBHHJGmm++G+CFF{SGG 33D$C3H**X__-,,w'  }}!#%//2<<Q?II!LVVWXY	%,,T\\2r2rJ	A%%//2<<Q?II!L	!4??,-33 % I -I<6C

O+D,R$
2(4OA|#t~~hQ'H&JJ#$$X=Qh%9W_`"(%!W-#A,"3G ")!Hh  !!'!*-+  50    9M(H!5S[\(*>hHM 1a(335H# %%jP]%j j %%_Q^%_ _)" #dn# 
 #"Zz"ZZ
cVX}j$Qccc$8=]g
 	
rE   )rq  rr  rs  rv   r   r'   r%   rn  rt  ro  rp  r$   r   )NNNNNNNNNNNNN)r   r   r   r   ru   r~  r  r  r  r  r   r  r  r   r   r[   Tensorboolr   r   rQ  r   r   r   r   s   @rC   rG  rG  G  s~   (#-"8 *  $L  -115'+,01515-1,004#',0/3&*~
ELL)~
 !.~
 u||$	~

 ELL)~
 !.~
 !.~
 U\\*~
 ELL)~
  -~
 4.~
 $D>~
 'tn~
 d^~
  
u&&	'!~
 ~
rE   rG  zt
    XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    )custom_introc            "       B  ^  \ rS rSrS/rU 4S jrS rS rSS jr\	              SS\
\R                     S\
\R                     S	\
\R                     S
\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\   S\
\   S\
\   S\
\   S\\\4   4S jj5       r\S	\\R                     S\R                  S\\R                     4S j5       rSrU =r$ )XLNetLMHeadModeli  zlm_loss.weightc                   > [         TU ]  U5        UR                  U l        UR                  U l        [	        U5      U l        [        R                  " UR                  UR                  SS9U l
        U R                  5         g )NTr  )rt   ru   rq  rp  rG  r   r	   r   rv   ru  r   rx  r   s     rC   ru   XLNetLMHeadModel.__init__  sb     ))!--%f-yy1B1BN 	rE   c                     U R                   $ r   r   r}  s    rC   get_output_embeddings&XLNetLMHeadModel.get_output_embeddings  s    ||rE   c                     Xl         g r   r  r  s     rC   set_output_embeddings&XLNetLMHeadModel.set_output_embeddings  s    %rE   c                 `  ^ UR                   S   n[        R                  " US4[        R                  UR                  S9nSmU(       a"  [        R
                  " US S 2T* S 24   U/SS9nO[        R
                  " X/SS9nUR                   S   n[        R                  " XWU4[        R                  UR                  S9nSUS S 2S S 2S4'   [        R                  " USU4[        R                  UR                  S9n	SU	S S 2SS4'   UUU	US.n
U(       a  [        U4S	 jU 5       5      U
S
'   U
$ )Nr   r   r  rq   r   r@  r   )r  r  r   r  c              3   @   >#    U  H  oS T* 2S S 2S S 24   v   M     g 7fr   rL  )r  
layer_pastoffsets     rC   r  AXLNetLMHeadModel.prepare_inputs_for_generation.<locals>.<genexpr>  s"     "`P_*hwh1n#=P_s   r   )rX   r[   r  r   r   r   r  r  )r   r  past_key_valuesr  r  effective_batch_sizedummy_tokensequence_lengthr  r   inputsr  s              @rC   prepare_inputs_for_generation.XLNetLMHeadModel.prepare_inputs_for_generation  s,   
  )q1kk#7";5::V_VfVfg
 		9Q[#9;"GQOI		9":BI $//!,KK!ODEKK`i`p`p
	 "	!Q( !1o6ekkR[RbRb
 $'q!Rx  #", 	
 ""`P_"``F6NrE   r  r  r   r  r   r  r  r   r  labelsr  r   r  r  r   c                    Ub  UOU R                   R                  nU R                  " U4UUUUUUUU	UUUUS.UD6nU R                  US   5      nSnU
bA  [	        5       nU" UR                  SUR                  S5      5      U
R                  S5      5      nU(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  UR                  S9$ )aI  
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
labels (`torch.LongTensor` of shape `(batch_size, num_predict)`, *optional*):
    Labels for masked language modeling. `num_predict` corresponds to `target_mapping.shape[1]`. If
    `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`.

    The labels should correspond to the masked input words that should be predicted and depends on
    `target_mapping`. Note in order to perform standard auto-regressive language modeling a *<mask>* token has
    to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples below)

    Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss
    is only computed for labels in `[0, ..., config.vocab_size]`
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.

Examples:

```python
>>> from transformers import AutoTokenizer, XLNetLMHeadModel
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-large-cased")
>>> model = XLNetLMHeadModel.from_pretrained("xlnet/xlnet-large-cased")

>>> # We show how to setup inputs to predict a next token using a bi-directional context.
>>> input_ids = torch.tensor(
...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
... ).unsqueeze(
...     0
... )  # We will predict the masked token
>>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
>>> perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token
>>> target_mapping = torch.zeros(
...     (1, 1, input_ids.shape[1]), dtype=torch.float
... )  # Shape [1, 1, seq_length] => let's predict one token
>>> target_mapping[
...     0, 0, -1
... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

>>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping)
>>> next_token_logits = outputs[
...     0
... ]  # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]

>>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling.
>>> input_ids = torch.tensor(
...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
... ).unsqueeze(
...     0
... )  # We will predict the masked token
>>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0)
>>> assert labels.shape[0] == 1, "only one word will be predicted"
>>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
>>> perm_mask[
...     :, :, -1
... ] = 1.0  # Previous tokens don't see last token as is done in standard auto-regressive lm training
>>> target_mapping = torch.zeros(
...     (1, 1, input_ids.shape[1]), dtype=torch.float
... )  # Shape [1, 1, seq_length] => let's predict one token
>>> target_mapping[
...     0, 0, -1
... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

>>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels)
>>> loss = outputs.loss
>>> next_token_logits = (
...     outputs.logits
... )  # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
```Nr  r   r  r   r  r  r   r  r  r   r  r  r   r   r   rW  rX  r   r   rS  )r9   r  r   r   r   viewr:  rV  r   r   rS  )r   r  r  r   r  r   r  r  r   r  r  r  r   r  r  r  transformer_outputsrX  rW  loss_fctr   s                        rC   r   XLNetLMHeadModel.forward  s   j &1%<k$++B]B]"..
)))!'/!5#
 
" 1!45')HFKKFKKO<fkk"oNDY!4QR!88F)-)9TGf$EvE%$))-;;*55
 	
rE   beam_idxc           	          U  Vs/ s H-  o"R                  SUR                  UR                  5      5      PM/     sn$ s  snf )z
This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
[`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
generation step.
r   )r   r  r   )r   r  r  s      rC   _reorder_cacheXLNetLMHeadModel._reorder_cache  s:     ^bb]az''8;;z7H7H+IJ]abbbs   4<)rq  r   rp  r   )NNNNNNNNNNNNNNNN)r   r   r   r   _tied_weights_keysru   r  r  r	  r   r   r[   r  r  r   r   rV  r   r   r   r  r   r   r   s   @rC   r  r    s    ++	&*X  -115'+,01515-1,004)-#',0/3&*Y
ELL)Y
 !.Y
 u||$	Y

 ELL)Y
 !.Y
 !.Y
 U\\*Y
 ELL)Y
  -Y
 &Y
 4.Y
 $D>Y
 'tnY
 d^Y
" 
u,,	-#Y
 Y
v cT%,,/ c5<< cDQVQ]Q]L^ c crE   r  z
    XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c            "         ^  \ rS rSrU 4S jr\              SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\	   S\\	   S\\	   S\\	   S\
\\4   4S jj5       rSrU =r$ )XLNetForSequenceClassificationi  c                   > [         TU ]  U5        UR                  U l        Xl        [	        U5      U l        [        U5      U l        [        R                  " UR                  UR                  5      U l        U R                  5         g r   )rt   ru   r.  r9   rG  r   r"  r   r	   r   rv   r   rx  r   s     rC   ru   'XLNetForSequenceClassification.__init__  se      ++%f- 4V <99V^^V5F5FG 	rE   r  r  r   r  r   r  r  r   r  r  r  r   r  r  r   c                 ~   Ub  UOU R                   R                  nU R                  " U4UUUUUUUU	UUUUS.UD6nUS   nU R                  U5      nU R	                  U5      nSnU
Gb  U R                   R
                  c  U R                  S:X  a  SU R                   l        OoU R                  S:  aN  U
R                  [        R                  :X  d  U
R                  [        R                  :X  a  SU R                   l        OSU R                   l        U R                   R
                  S:X  aJ  [        5       nU R                  S:X  a&  U" UR                  5       U
R                  5       5      nOU" UU
5      nOU R                   R
                  S:X  a=  [        5       nU" UR                  SU R                  5      U
R                  S5      5      nO-U R                   R
                  S:X  a  [        5       nU" UU
5      nU(       d  U4USS -   nUb  U4U-   $ U$ [!        UUUR"                  UR$                  UR&                  S	9$ )
a6	  
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.
Nr  r   r   
regressionsingle_label_classificationmulti_label_classificationr   r  )r9   r  r   r   r   problem_typer.  r   r[   r   intr   r   r   r  r
   r[  r   r   rS  )r   r  r  r   r  r   r  r  r   r  r  r  r   r  r  r  r  r   rX  rW  r  s                        rC   r   &XLNetForSequenceClassification.forward  s   r &1%<k$++B]B]"..
)))!'/!5#
 
  %Q'&&v.!!&){{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y!4QR!88F)-)9TGf$EvE3$))-;;*55
 	
rE   )r9   r   r.  r   r   r  )r   r   r   r   ru   r   r   r[   r  r  r   r   r[  r   r   r   r   s   @rC   r  r    sb   
  -115'+,01515-1,004)-#',0/3&*p
ELL)p
 !.p
 u||$	p

 ELL)p
 !.p
 !.p
 U\\*p
 ELL)p
  -p
 &p
 4.p
 $D>p
 'tnp
 d^p
" 
u::	;#p
 p
rE   r  c            "         ^  \ rS rSrU 4S jr\              SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\	   S\\	   S\\	   S\\	   S\
\\4   4S jj5       rSrU =r$ )XLNetForTokenClassificationi<  c                    > [         TU ]  U5        UR                  U l        [        U5      U l        [
        R                  " UR                  UR                  5      U l        U R                  5         g r   )
rt   ru   r.  rG  r   r	   r   r   
classifierrx  r   s     rC   ru   $XLNetForTokenClassification.__init__>  T      ++%f-))F$6$68I8IJ 	rE   r  r  r   r  r   r  r  r   r  r  r  r   r  r  r   c                    Ub  UOU R                   R                  nU R                  UUUUUUUUU	UUUUS9nUS   nU R                  U5      nSnU
b<  [	        5       nU" UR                  SU R                  5      U
R                  S5      5      nU(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  UR                  S9$ )a	  
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
    where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.emory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.
Nr  r   r   r   r  )r9   r  r   r'  r   r  r.  r^  r   r   rS  )r   r  r  r   r  r   r  r  r   r  r  r  r   r  r  r  r   sequence_outputrX  rW  r  r   s                         rC   r   #XLNetForTokenClassification.forwardH  s    t &1%<k$++B]B]"")))!'/!5# # 
  "!*1')HFKKDOO<fkk"oNDY,F)-)9TGf$EvE0!//))
 	
rE   )r'  r.  r   r  )r   r   r   r   ru   r   r   r[   r  r  r   r   r^  r   r   r   r   s   @rC   r%  r%  <  sb     -115'+,01515-1,004)-#',0/3&*^
ELL)^
 !.^
 u||$	^

 ELL)^
 !.^
 !.^
 U\\*^
 ELL)^
  -^
 &^
 4.^
 $D>^
 'tn^
 d^^
" 
u77	8#^
 ^
rE   r%  c            "         ^  \ rS rSrU 4S jr\              SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\	   S\\	   S\\	   S\\	   S\
\\4   4S jj5       rSrU =r$ )XLNetForMultipleChoicei  c                    > [         TU ]  U5        [        U5      U l        [	        U5      U l        [        R                  " UR                  S5      U l	        U R                  5         g r   )rt   ru   rG  r   r"  r   r	   r   rv   r   rx  r   s     rC   ru   XLNetForMultipleChoice.__init__  sM     %f- 4V <99V^^Q7 	rE   r  r  r  r  r   r  r   r   r  r  r  r   r  r  r   c                    Ub  UOU R                   R                  nUb  UR                  S   OU	R                  S   nUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnU	b1  U	R                  SU	R	                  S5      U	R	                  S5      5      OSnU R
                  " U4UUUUUUUUUUUUS.UD6nUS   nU R                  U5      nU R                  U5      nUR                  SU5      nSnU
b"  [        5       nU" UU
R                  S5      5      nU(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  UR                  S9$ )a  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
input_mask (`torch.FloatTensor` of shape `batch_size, num_choices, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.
Nr   r   r  )r  r  r  r   r  r   r   r  r  r   r  r  r   r  )r9   r  rX   r  r:  r   r   r   r   r`  r   r   rS  )r   r  r  r  r  r   r  r   r   r  r  r  r   r  r  r  num_choicesflat_input_idsflat_token_type_idsflat_attention_maskflat_input_maskflat_inputs_embedsr  r   rX  reshaped_logitsrW  r  s                               rC   r   XLNetForMultipleChoice.forward  s   T &1%<k$++B]B],5,Aiooa(}GZGZ[\G]CLCXINN2,>?^bR`Rln11"n6I6I"6MNrvR`Rln11"n6I6I"6MNrvFPF\*//"joob.ABbf ( r=#5#5b#9=;M;Mb;QR 	 #..
.&.),/!5#
 
" %Q'&&v.!!&) ++b+6')HOV[[_=D%'*=ab*AAF)-)9TGf$EvE+"$))-;;*55
 	
rE   )r   r   r   r  )r   r   r   r   ru   r   r   r[   r  r  r   r   r`  r   r   r   r   s   @rC   r.  r.    sb     -115-115'+,015,004)-#',0/3&*}
ELL)}
 !.}
 U\\*	}

 !.}
 u||$}
 ELL)}
 !.}
 ELL)}
  -}
 &}
 4.}
 $D>}
 'tn}
 d^}
" 
u22	3#}
 }
rE   r.  z
    XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c            $         ^  \ rS rSrU 4S jr\               SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\R                     S\\	   S\\	   S\\	   S\\	   S\
\\4   4 S jj5       rSrU =r$ )XLNetForQuestionAnsweringSimplei7  c                    > [         TU ]  U5        UR                  U l        [        U5      U l        [
        R                  " UR                  UR                  5      U l        U R                  5         g r   )
rt   ru   r.  rG  r   r	   r   r   
qa_outputsrx  r   s     rC   ru   (XLNetForQuestionAnsweringSimple.__init__>  r)  rE   r  r  r   r  r   r  r  r   r  r  end_positionsr  r   r  r  r   c                 N   Ub  UOU R                   R                  nU R                  " U4UUUUUUUU	UUUUS.UD6nUS   nU R                  U5      nUR	                  SSS9u  nnUR                  S5      R                  5       nUR                  S5      R                  5       nSnU
b  Ub  [        U
R                  5       5      S:  a  U
R                  S5      n
[        UR                  5       5      S:  a  UR                  S5      nUR                  S5      nU
R                  SU5      n
UR                  SU5      n[        US9nU" UU
5      nU" UU5      nUU-   S-  nU(       d  UU4USS -   nUb  U4U-   $ U$ [        UUUUR                  UR                  UR                  S	9$ )
r  Nr  r   r   r   r   )ignore_indexrq   )rW  rc  rd  r   r   rS  )r9   r  r   r=  splitr   r  rW   r:  r  r   rb  r   r   rS  )r   r  r  r   r  r   r  r  r   r  r  r?  r  r   r  r  r  r   r+  rX  rc  rd  
total_lossignored_indexr  
start_lossend_lossr   s                               rC   r   'XLNetForQuestionAnsweringSimple.forwardH  s   l &1%<k$++B]B]""
)))!'/!5#
 
" "!*1#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J"J/'!"+=F/9/EZMF*Q6Q4%!!//))
 	
rE   )r.  r=  r   )NNNNNNNNNNNNNNN)r   r   r   r   ru   r   r   r[   r  r  r   r   rb  r   r   r   r   s   @rC   r;  r;  7  sy     -115'+,01515-1,0042604#',0/3&*!k
ELL)k
 !.k
 u||$	k

 ELL)k
 !.k
 !.k
 U\\*k
 ELL)k
  -k
 "%,,/k
  -k
 4.k
 $D>k
 'tnk
  d^!k
$ 
u;;	<%k
 k
rE   r;  c            *       >  ^  \ rS rSrU 4S jr\                  SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\	   S\\	   S\\	   S\\	   S\
\\4   4&S jj5       rSrU =r$ )XLNetForQuestionAnsweringi  c                   > [         TU ]  U5        UR                  U l        UR                  U l        [	        U5      U l        [        U5      U l        [        U5      U l	        [        U5      U l        U R                  5         g r   )rt   ru   start_n_top	end_n_toprG  r   r   rc  r  rd  r  answer_classrx  r   s     rC   ru   "XLNetForQuestionAnswering.__init__  sj     !--))%f-26:.v626: 	rE   r  r  r   r  r   r  r  r   r  r  r?  is_impossibler  r   r  r   r  r  r   c                 l   Ub  UOU R                   R                  nU R                  " U4UUUUUUUU	UUUUS.UD6nUS   nU R                  UUS9nUSS nU
b  Ub  XX4 H/  nUc  M  UR	                  5       S:  d  M  UR                  S5        M1     U R                  UXS9n[        5       nU" UU
5      nU" UU5      nUU-   S-  nUb9  Ub6  U R                  UXS	9n[        R                  " 5       nU" UU5      n UU S
-  -  nU(       d	  U4USS -   $ [        UUR                  UR                  UR                  S9$ UR                  5       u  n!n"n#[        R                   R#                  USS9n$[$        R&                  " U$U R(                  SS9u  n%n&U&R+                  S5      R-                  SSU#5      n'[$        R.                  " USU'5      n(U(R+                  S5      R-                  SU"SS5      n(UR+                  S5      R1                  U(5      n)Ub  UR+                  S5      OSnU R                  U)U(US9n[        R                   R#                  USS9n*[$        R&                  " U*U R2                  SS9u  n+n,U+R5                  SU R(                  U R2                  -  5      n+U,R5                  SU R(                  U R2                  -  5      n,[$        R6                  " SUU$5      n(U R                  UU(US9nU(       d  U%U&U+U,U4nUUSS -   $ [        U%U&U+U,UUR                  UR                  UR                  S9$ )a_  
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
    Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
    decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
    they have already been computed.

    `use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
    Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

    - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
    - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

    If not set, each token attends to all the others (full bidirectional attention). Only used during
    pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
    Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
    on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
    (generation).
input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
    Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
    real tokens and 1 for padding which is kept for compatibility with the original code base.

    Mask values selected in `[0, 1]`:

    - 1 for tokens that are **masked**,
    - 0 for tokens that are **not masked**.

    You can only uses one of `input_mask` and `attention_mask`.
is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels whether a question has an answer or no answer (SQuAD 2.0)
cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for position (index) of the classification token to use as input for computing plausibility of the
    answer.
p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
    masked. 0.0 mean token is not masked.
use_mems (`bool`, *optional*):
    Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
    states from previous forward passes to compute attention, which can significantly improve performance for
    sequential decoding tasks.

Example:

```python
>>> from transformers import AutoTokenizer, XLNetForQuestionAnswering
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-base-cased")
>>> model = XLNetForQuestionAnswering.from_pretrained("xlnet/xlnet-base-cased")

>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
...     0
... )  # Batch size 1
>>> start_positions = torch.tensor([1])
>>> end_positions = torch.tensor([3])
>>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)

>>> loss = outputs.loss
```Nr  r   )r   r   r   )r  r   rq   )r  r  rp   )rW  r   r   rS  r   r  )r  r   z
blh,bl->bh)r  r  )rg  rh  ri  rj  rk  r   r   rS  )r9   r  r   rc  r   squeeze_rd  r   rM  r	   r
   rf  r   r   rS  r:  r   r   r[   topkrK  r9  r  r  	expand_asrL  r  r   )-r   r  r  r   r  r   r  r  r   r  r  r?  rO  r  r   r  r   r  r  r  r  r   rc  r   r   rd  r  rE  rF  rC  rk  loss_fct_clscls_lossr  r  r  start_log_probsrg  rh  start_top_index_expr  hidden_states_expandedend_log_probsri  rj  s-                                                rC   r   !XLNetForQuestionAnswering.forward  s   f &1%<k$++B]B]"..
)))!'/!5#
 
  ,A.((v(F%ab)&=+D%iO=QUUWq[JJrN P
 gJ')H!,@J
M:H$x/14J$)B!..}o.s
!335'
MB hn,
"}':12'>>>6#,11"5"C"C2==	  +//1NCs mm33Lb3IO38::!1!1r40 #2";";B"?"F"Fr2s"S <<r;NOL'11!4;;Bb"ML%2%<%<Q%?%I%I&" .4-?V%%b)TF)?lcijJMM11*!1DM/4zzt~~10,} !2 6 6r4;K;Kdnn;\ ])..r43C3Cdnn3TUM <<m_L **LI + J .ARTacmn!4QR!8886(;$3&7"/),11"5"C"C2==	 	rE   )rM  rd  rL  rc  rK  r   )NNNNNNNNNNNNNNNNNN)r   r   r   r   ru   r   r   r[   r  r  r   r   rf  r   r   r   r   s   @rC   rI  rI    s     -115'+,01515-1,004260404,0)-#',0/3&*'wELL)w !.w u||$	w
 ELL)w !.w !.w U\\*w ELL)w  -w "%,,/w  -w  -w ELL)w &w  4.!w" $D>#w$ 'tn%w& d^'w* 
u55	6+w wrE   rI  )	r.  rI  r;  r  r%  r  rG  r=  rk   r   )<r  r  dataclassesr   typingr   r   r   r   r   r[   r	   torch.nnr
   r   r   activationsr   r   
generationr   modeling_utilsr   pytorch_utilsr   utilsr   r   r   configuration_xlnetr   
get_loggerr   rM   rD   rk   Modulerm   r   r   r   r  r  r"  r=  rQ  rV  r[  r^  r`  rb  rf  rG  r  r  r%  r.  r;  rI  __all__rL  rE   rC   <module>rg     s     ! 9 9   A A 1 ) - 6 9 9 , 
		H	%Pf@FARYY AHryy ./ /f#RYY #NB299 BL>RYY >D`299 `F "V? "V "VJ ?{ ? ?B !?[ !? !?H ?; ? ?B ? ? ?B  ?;  ?  ?F !?K !? !?H ,?k ,? ,?^ 
% 
 
D 
cc+_ cc
ccL ~
%9 ~
~
B j
"6 j
 j
Z I
1 I
 I
X w
&: w
w
t F 4 F FR
rE   