
    fThb                        S r SSKrSSKJr  SSKJrJrJrJr  SSK	r	SSK
r	SSK	Jr  SSKJr  SSKJr  SS	KJrJrJr  SS
KJr  SSKJrJrJr  SSKJrJrJr  SSKJr  \R@                  " \!5      r"\ " S S\5      5       r# " S S\RH                  5      r% " S S\RH                  5      r& " S S\RH                  5      r' " S S\RH                  5      r( " S S\RH                  5      r) " S S\RH                  5      r* " S S\RH                  5      r+ " S S \RH                  5      r, " S! S"\RH                  5      r- " S# S$\RH                  5      r. " S% S&\RH                  5      r/ " S' S(\RH                  5      r0 " S) S*\RH                  5      r1\ " S+ S,\5      5       r2\ " S- S.\25      5       r3\ " S/ S0\25      5       r4\" S1S29 " S3 S4\25      5       r5\" S5S29 " S6 S7\25      5       r6/ S8Qr7g)9zPyTorch Bros model.    N)	dataclass)ListOptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputauto_docstringlogging   )
BrosConfigc                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   S	rg)
BrosSpadeOutput)   a  
Base class for outputs of token classification models.

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) :
        Classification loss.
    initial_token_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
        Classification scores for entity initial tokens (before SoftMax).
    subsequent_token_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length+1)`):
        Classification scores for entity sequence tokens (before SoftMax).
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
Nlossinitial_token_logitssubsequent_token_logitshidden_states
attentions )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations__r   r   r   r   r   __static_attributes__r        ^/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/bros/modeling_bros.pyr   r   )   s~    . )-D(5$$
%,8<(5#4#45<;?Xe&7&78?8<M8E%"3"345<59Ju00129r*   r   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )BrosPositionalEmbedding1DI   c                    > [         [        U ]  5         UR                  U l        SS[        R
                  " SU R                  S5      U R                  -  -  -  nU R                  SU5        g )Nr   i'          g       @inv_freq)superr-   __init__dim_bbox_sinusoid_emb_1dr&   arangeregister_buffer)selfconfigr1   	__class__s      r+   r3   "BrosPositionalEmbedding1D.__init__L   sa    '79(.(G(G%ell3(E(EsKdNkNkkl
 	Z2r*   pos_seqreturnc                    UR                  5       nUu  p4nUR                  X4US5      U R                  R                  SSSU R                  S-  5      -  n[        R
                  " UR                  5       UR                  5       /SS9nU$ )Nr      dim)sizeviewr1   r4   r&   catsincos)r7   r;   seq_sizeb1b2b3sinusoid_inppos_embs           r+   forward!BrosPositionalEmbedding1D.forwardV   s{    <<>
||BB2T]]5G5G1aQUQnQnrsQs5tt))\--/1A1A1CD"Mr*   )r4   
r!   r"   r#   r$   r3   r&   TensorrM   r)   __classcell__r9   s   @r+   r-   r-   I   s(    3u||   r*   r-   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )BrosPositionalEmbedding2D^   c                    > [         [        U ]  5         UR                  U l        [	        U5      U l        [	        U5      U l        g N)r2   rT   r3   dim_bboxr-   	x_pos_emb	y_pos_embr7   r8   r9   s     r+   r3   "BrosPositionalEmbedding2D.__init___   s4    '7926:26:r*   bboxr<   c                    / n[        U R                  5       HX  nUS-  S:X  a'  UR                  U R                  USU4   5      5        M3  UR                  U R	                  USU4   5      5        MZ     [
        R                  " USS9nU$ )Nr>   r   .r?   r@   )rangerX   appendrY   rZ   r&   rD   )r7   r]   stackibbox_pos_embs        r+   rM   !BrosPositionalEmbedding2D.forwardf   sw    t}}%A1uzT^^DaL9:T^^DaL9:	 &
 yyB/r*   )rX   rY   rZ   rO   rR   s   @r+   rT   rT   ^   s(    ;ELL U\\  r*   rT   c                   J   ^  \ rS rSrU 4S jrS\R                  4S jrSrU =r	$ )BrosBboxEmbeddingsq   c                    > [         [        U ]  5         [        U5      U l        [
        R                  " UR                  UR                  SS9U l	        g )NF)bias)
r2   rf   r3   rT   bbox_sinusoid_embr   Lineardim_bbox_sinusoid_emb_2ddim_bbox_projectionbbox_projectionr[   s     r+   r3   BrosBboxEmbeddings.__init__r   s@     $02!:6!B!yy)H)H&JdJdkpqr*   r]   c                     UR                  SS5      nUS S S 2S S 2S S 24   US S 2S S S 2S S 24   -
  nU R                  U5      nU R                  U5      nU$ )Nr   r   )	transposerj   rn   )r7   r]   bbox_tbbox_posrc   s        r+   rM   BrosBboxEmbeddings.forwardw   s\    1%$1a-(6!T1a-+@@--h7++L9r*   )rn   rj   rO   rR   s   @r+   rf   rf   q   s    r
ELL  r*   rf   c                      ^  \ rS rSrSrU 4S jr     SS\\R                     S\\R                     S\\R                     S\\R                     S\	S	\R                  4S
 jjr
SrU =r$ )BrosTextEmbeddings   zGConstruct the embeddings from word, position and token_type embeddings.c           	      Z  > [         TU ]  5         [        R                  " UR                  UR
                  UR                  S9U l        [        R                  " UR                  UR
                  5      U l	        [        R                  " UR                  UR
                  5      U l        [        R                  " UR
                  UR                  S9U l        [        R                  " UR                  5      U l        [#        USS5      U l        U R'                  S[(        R*                  " UR                  5      R-                  S5      5        U R'                  S[(        R.                  " U R0                  R3                  5       [(        R4                  U R0                  R6                  S9S	S
9  g )N)padding_idxepsposition_embedding_typeabsoluteposition_ids)r   r?   token_type_idsdtypedeviceF)
persistent)r2   r3   r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutgetattrr|   r6   r&   r5   expandzerosr~   rB   longr   r[   s     r+   r3   BrosTextEmbeddings.__init__   s8   !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]" f&8&8f>S>STzz&"<"<='.v7PR\']$^U\\&:X:X-Y-`-`ah-ijKK!!&&(jj((//
  	 	
r*   	input_idsr   r~   inputs_embedspast_key_values_lengthr<   c                 d   Ub  UR                  5       nOUR                  5       S S nUS   nUc  U R                  S S 2XWU-   24   nUcv  [        U S5      (       a-  U R                  S S 2S U24   nUR	                  US   U5      n	U	nO8[
        R                  " U[
        R                  U R                  R                  S9nUc  U R                  U5      nU R                  U5      n
XJ-   nU R                  S:X  a  U R                  U5      nX-  nU R                  U5      nU R                  U5      nU$ )Nr?   r   r   r   r   r}   )rB   r~   hasattrr   r   r&   r   r   r   r   r   r|   r   r   r   )r7   r   r   r~   r   r   input_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   s                r+   rM   BrosTextEmbeddings.forward   sA     #..*K',,.s3K ^
,,Q0FVlIl0l-lmL!t-..*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[

SWSdSdSkSk!l  00;M $ : :> J":
'':5"&":":<"H-J^^J/
\\*-
r*   )r   r   r|   r   r   r   )NNNNr   )r!   r"   r#   r$   r%   r3   r   r&   rP   intrM   r)   rQ   rR   s   @r+   rv   rv      s    Q
4 -115/304&'$ELL)$ !.$ u||,	$
  -$ !$$ 
$ $r*   rv   c                   r  ^  \ rS rSrU 4S jrS\R                  4S jr      SS\R                  S\R                  S\\R                     S\\R                     S	\\R                     S
\\R                     S\\	\	\R                           S\\R                     S\	\R                     4S jjrSrU =r$ )BrosSelfAttention   c                   > [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eUR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  5      U l        [#        USS5      U l        U R$                  S:X  d  U R$                  S	:X  aG  UR&                  U l        [        R(                  " S
UR&                  -  S-
  U R                  5      U l        UR,                  U l        g )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r|   r}   relative_keyrelative_key_queryr>   r   )r2   r3   r   num_attention_headsr   
ValueErrorr   attention_head_sizeall_head_sizer   rk   querykeyvaluer   attention_probs_dropout_probr   r   r|   r   r   distance_embedding
is_decoderr[   s     r+   r3   BrosSelfAttention.__init__   s    : ::a?PVXhHiHi#F$6$6#7 8 445Q8 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF'.v7PR\']$''>9T=Y=Y]q=q+1+I+ID(&(ll1v7U7U3UXY3Y[_[s[s&tD# ++r*   xc                     UR                  5       S S U R                  U R                  4-   nUR                  " U6 nUR	                  SSSS5      $ )Nr?   r   r>   r   r
   )rB   r   r   rC   permute)r7   r   new_x_shapes      r+   transpose_for_scores&BrosSelfAttention.transpose_for_scores   sT    ffhsm$$$$'
 
 FFK yyAq!$$r*   r   rc   attention_mask	head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsr<   c	                 B   U R                  U5      n	US Ln
U
(       a  Ub  US   nUS   nUnGOU
(       aC  U R                  U R                  U5      5      nU R                  U R                  U5      5      nUnOUbu  U R                  U R                  U5      5      nU R                  U R                  U5      5      n[        R
                  " US   U/SS9n[        R
                  " US   U/SS9nO@U R                  U R                  U5      5      nU R                  U R                  U5      5      nU R                  U	5      nU R                  (       a  X4n[        R                  " XR                  SS5      5      nU R                  S:X  d  U R                  S:X  GaD  UR                  5       S   n[        R                  " U[        R                  UR                  S	9R                  SS5      n[        R                  " U[        R                  UR                  S	9R                  SS5      nUU-
  nU R                  UU R                   -   S-
  5      nUR#                  UR$                  S
9nU R                  S:X  a  [        R&                  " SUU5      nUU-   nOHU R                  S:X  a8  [        R&                  " SUU5      n[        R&                  " SUU5      nUU-   U-   nUR(                  u  nnnnUR                  XUU5      nUR+                  / SQ5      n[        R&                  " SX45      nUU-   nU[,        R.                  " U R0                  5      -  nUb  X-   n[2        R4                  " SS9" U5      nU R7                  U5      nUb  UU-  n[        R                  " UU5      nUR+                  SSSS5      R9                  5       nUR                  5       S S U R:                  4-   nUR                  " U6 nU(       a  UU4OU4nU R                  (       a  UU4-   nU$ )Nr   r   r>   r@   r?   r   r   r   )r   zbhld,lrd->bhlrzbhrd,lrd->bhlr)r>   r   r   r
   zbnid,bijd->bnijr
   )r   r   r   r   r&   rD   r   matmulrq   r|   rB   r5   r   r   rC   r   r   tor   einsumshaper   mathsqrtr   r   Softmaxr   
contiguousr   )r7   r   rc   r   r   r   r   r   r   mixed_query_layeris_cross_attention	key_layervalue_layerquery_layerattention_scoresr   position_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_key
batch_sizen_headd_headbbox_pos_scoresattention_probscontext_layernew_context_layer_shapeoutputss                                  r+   rM   BrosSelfAttention.forward   s    !JJ}5
 3$>."<&q)I(+K3N11$((;P2QRI33DJJ?T4UVK3N'11$((=2IJI33DJJ}4MNK		>!#4i"@aHI))^A%6$D!LK11$((=2IJI33DJJ}4MNK//0AB?? (5N !<<5H5HR5PQ''>9T=Y=Y]q=q&++-a0J"\\*EJJ}OcOcdiijlnopN"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||<LiYm/n,#36T#TWs#s  2=1B1B.
FJ#((VT#++L9,,'8;:UV+o=+dii8P8P.QQ%/@ **,-=> ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**,CD6G=/2mM]?? 11Gr*   )r   r   r   r   r   r   r   r   r|   r   r   NNNNNF)r!   r"   r#   r$   r3   r&   rP   r   r   r   r'   rM   r)   rQ   rR   s   @r+   r   r      s    ,0%ell % 26,08<9=DH49f||f llf !.	f
 ELL)f  (5f !) 6f !uU->->'?!@Af $ELL1f 
u||	f fr*   r   c                   z   ^  \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )BrosSelfOutputiM  c                 (  > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  5      U l
        g Nrz   )r2   r3   r   rk   r   denser   r   r   r   r   r[   s     r+   r3   BrosSelfOutput.__init__N  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=r*   r   input_tensorr<   c                 p    U R                  U5      nU R                  U5      nU R                  X-   5      nU$ rW   r   r   r   r7   r   r   s      r+   rM   BrosSelfOutput.forwardT  5    

=1]3}'CDr*   r   r   r   rO   rR   s   @r+   r   r   M  6    >U\\  RWR^R^  r*   r   c                   B  ^  \ rS rSrU 4S jrS r      SS\R                  S\R                  S\\R                     S\\R                     S\\R                     S	\\R                     S
\\	\	\R                           S\\   S\	\R                     4S jjrSrU =r$ )BrosAttentioni[  c                    > [         TU ]  5         [        U5      U l        [	        U5      U l        [        5       U l        g rW   )r2   r3   r   r7   r   outputsetpruned_headsr[   s     r+   r3   BrosAttention.__init__\  s0    %f-	$V,Er*   c                 8   [        U5      S:X  a  g [        UU R                  R                  U R                  R                  U R
                  5      u  p[        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l	        [        U R                  R                  USS9U R                  l        U R                  R                  [        U5      -
  U R                  l        U R                  R                  U R                  R                  -  U R                  l        U R
                  R                  U5      U l        g )Nr   r   r@   )lenr   r7   r   r   r   r   r   r   r   r   r   r   union)r7   headsindexs      r+   prune_headsBrosAttention.prune_headsb  s   u:?7II))II))	
 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:r*   r   rc   r   r   r   r   r   r   r<   c	                 n    U R                  UUUUUUUUS9n	U R                  U	S   U5      n
U
4U	SS  -   nU$ )Nr   rc   r   r   r   r   r   r   r   r   )r7   r   )r7   r   rc   r   r   r   r   r   r   self_outputsattention_outputr   s               r+   rM   BrosAttention.forwardw  s_     yy'%)"7#9)/ ! 	
  ;;|AF#%QR(88r*   )r   r   r7   r   )r!   r"   r#   r$   r3   r   r&   rP   r   r   r'   boolrM   r)   rQ   rR   s   @r+   r   r   [  s    ";2 26,08<9=DH,1|| ll !.	
 ELL)  (5 !) 6 !uU->->'?!@A $D> 
u||	 r*   r   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )BrosIntermediatei  c                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g rW   )r2   r3   r   rk   r   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnr[   s     r+   r3   BrosIntermediate.__init__  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r*   r   r<   c                 J    U R                  U5      nU R                  U5      nU$ rW   r   r  )r7   r   s     r+   rM   BrosIntermediate.forward  s&    

=100?r*   r  rO   rR   s   @r+   r  r    s(    9U\\ ell  r*   r  c                   z   ^  \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )
BrosOutputi  c                 (  > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR
                  UR                  S9U l        [        R                  " UR                  5      U l        g r   )r2   r3   r   rk   r  r   r   r   r   r   r   r   r[   s     r+   r3   BrosOutput.__init__  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=r*   r   r   r<   c                 p    U R                  U5      nU R                  U5      nU R                  X-   5      nU$ rW   r   r   s      r+   rM   BrosOutput.forward  r   r*   r   rO   rR   s   @r+   r  r    r   r*   r  c                   B  ^  \ rS rSrU 4S jr      SS\R                  S\R                  S\\R                     S\\R                     S\\R                     S\\R                     S	\\	\	\R                           S
\\
   S\	\R                     4S jjrS rSrU =r$ )	BrosLayeri  c                 v  > [         TU ]  5         UR                  U l        SU l        [	        U5      U l        UR                  U l        UR                  U l        U R                  (       a/  U R                  (       d  [        U  S35      e[	        U5      U l	        [        U5      U l        [        U5      U l        g )Nr   z> should be used as a decoder model if cross attention is added)r2   r3   chunk_size_feed_forwardseq_len_dimr   	attentionr   add_cross_attention	Exceptioncrossattentionr  intermediater  r   r[   s     r+   r3   BrosLayer.__init__  s    '-'E'E$&v. ++#)#=#= ##??4&(f ghh"/"7D,V4 (r*   r   rc   r   r   r   r   r   r   r<   c	           	         Ub  US S OS n	U R                  UUUUUU	S9n
U
S   nU R                  (       a  U
SS nU
S   nOU
SS  nS nU R                  (       a[  UbX  [        U S5      (       a  [        SU  S35      eUb  US	S  OS nU R	                  UUUUUUU5      nUS   nUUSS -   nUS   nWU-   n[        U R                  U R                  U R                  U5      nU4U-   nU R                  (       a  UW4-   nU$ )
Nr>   )rc   r   r   r   r   r   r   r?   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r   )	r  r   r   r  r  r   feed_forward_chunkr  r  )r7   r   rc   r   r   r   r   r   r   self_attn_past_key_valueself_attention_outputsr  r   present_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputslayer_outputs                     r+   rM   BrosLayer.forward  s    :H9S>"1#5Y] !%%)/3 "0 "
 2!4 ??,Qr2G 6r :,QR0G'+$??4@t-..=dV  Dd  e 
 @N?Yrs(;_c%&*&9&9 %&)!'#  7q9 7" ==G ,C2+F( 14P P0##((	
  /G+ ??!2 44Gr*   c                 J    U R                  U5      nU R                  X!5      nU$ rW   )r   r   )r7   r  intermediate_outputr*  s       r+   r#  BrosLayer.feed_forward_chunk  s)    "//0@A{{#6Ir*   )r  r  r  r  r   r   r   r  r   )r!   r"   r#   r$   r3   r&   rP   r   r'   r   r  rM   r#  r)   rQ   rR   s   @r+   r  r    s    )$ 7;15=A>BDH,1C||C llC !!2!23	C
 E--.C  ((9(9:C !)):): ;C !uU->->'?!@AC $D>C 
u||	CJ r*   r  c                   j  ^  \ rS rSrU 4S jr         SS\R                  S\R                  S\\R                     S\\R                     S\\R                     S\\R                     S	\\	\	\R                           S
\\
   S\\
   S\\
   S\\
   S\\	\R                     \4   4S jjrSrU =r$ )BrosEncoderi	  c                    > [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        g s  snf rW   )	r2   r3   r8   r   
ModuleListr_   num_hidden_layersr  layer)r7   r8   _r9   s      r+   r3   BrosEncoder.__init__
  sG    ]]uVE]E]?^#_?^!If$5?^#_`
#_s   Ar   rc   r   r   r   r   past_key_values	use_cacher   output_hidden_statesreturn_dictr<   c                    U
(       a  SOS nU	(       a  SOS nU	(       a  U R                   R                  (       a  SOS nU(       a  SOS n[        U R                  5       H  u  nnU
(       a  X4-   nUb  UU   OS nUb  UU   OS n[	        U R                   SS5      (       aR  U R
                  (       aA  U(       a  [        R                  S5        SnU R                  UR                  UUUUUUU	5      nOU" UUUUUUUU	S9nUS   nU(       a	  UUS   4-  nU	(       d  M  UUS   4-   nU R                   R                  (       d  M  UUS	   4-   nM     U
(       a  X4-   nU(       d  [        S
 UUUUU4 5       5      $ [        UUUUUS9$ )Nr    gradient_checkpointingFzh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...r   r   r?   r   r>   c              3   0   #    U  H  nUc  M  Uv   M     g 7frW   r    ).0vs     r+   	<genexpr>&BrosEncoder.forward.<locals>.<genexpr>R  s"      
A  s   	)last_hidden_stater7  r   r   cross_attentions)r8   r  	enumerater4  r   trainingloggerwarning_gradient_checkpointing_func__call__tupler   )r7   r   rc   r   r   r   r   r7  r8  r   r9  r:  all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cacherb   layer_modulelayer_head_maskr   layer_outputss                        r+   rM   BrosEncoder.forward  s    #7BD$5b4%64;;;Z;Zr`d#,R$(4OA|#$58H$H!.7.CilO3B3N_Q/TXNt{{$<eDDNN/ !&I $ A A ))! "#)*%	! !-"/!-#1-*?+A#1&7	! *!,M"}R'8&::"  &9]1=M<O&O#;;222+?=QRCSBU+U(U  5X   14D D 
 "&%'(
 
 
 9+.+*1
 	
r*   )r8   r4  )	NNNNNNFFT)r!   r"   r#   r$   r3   r&   rP   r   r'   r   r  r   r   rM   r)   rQ   rR   s   @r+   r0  r0  	  s!   a 7;15=A>BEI$(,1/4&*T
||T
 llT
 !!2!23	T

 E--.T
  ((9(9:T
 !)):): ;T
 "%e.?.?(@"ABT
 D>T
 $D>T
 'tnT
 d^T
 
uU\\"$MM	NT
 T
r*   r0  c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )
BrosPoolerig  c                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " 5       U l        g rW   )r2   r3   r   rk   r   r   Tanh
activationr[   s     r+   r3   BrosPooler.__init__h  s9    YYv1163E3EF
'')r*   r   r<   c                 \    US S 2S4   nU R                  U5      nU R                  U5      nU$ )Nr   )r   rW  )r7   r   first_token_tensorpooled_outputs       r+   rM   BrosPooler.forwardm  s6     +1a40

#566r*   )rW  r   rO   rR   s   @r+   rT  rT  g  s(    $
U\\ ell  r*   rT  c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )BrosRelationExtractoriv  c                 \  > [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        UR                  U l        [        R                  " U R                  5      U l	        [        R                  " U R                  U R                  U R
                  -  5      U l        [        R                  " U R                  U R                  U R
                  -  5      U l        [        R                  " [        R                  " SU R                  5      5      U l        g )Nr   )r2   r3   n_relationsr   backbone_hidden_sizehead_hidden_sizeclassifier_dropout_probr   r   droprk   r   r   	Parameterr&   r   
dummy_noder[   s     r+   r3   BrosRelationExtractor.__init__w  s    !--$*$6$6! & 2 2'-'E'E$JJt;;<	YYt88$:J:JTMbMb:bc
99T668H8H4K`K`8`a,,u{{1d6O6O'PQr*   r   r   c           	         U R                  U R                  U5      5      nU R                  R                  S5      R	                  SUR                  S5      S5      n[        R                  " X#/SS9nU R                  U R                  U5      5      nUR                  UR                  S5      UR                  S5      U R                  U R                  5      nUR                  UR                  S5      UR                  S5      U R                  U R                  5      n[        R                  " UR                  SSSS5      UR                  SSSS5      5      nU$ )Nr   r   axisr>   r
   )r   rd  rf  	unsqueezerepeatrB   r&   rD   r   rC   r`  rb  r   r   )r7   r   r   	dummy_vecrelation_scores        r+   rM   BrosRelationExtractor.forward  s   jj;!78OO--a0779>>!;LaP	IIy41=	HHTYYy12	!&&Q!1!1!!4d6F6FH]H]
 NN9>>!#4innQ6GIYIY[_[p[pq	1a+Y->->q!Q-J
 r*   )ra  rc  rd  rf  rb  r   r`  r   rO   rR   s   @r+   r^  r^  v  s)    R5<< ELL  r*   r^  c                   "    \ rS rSr\rSrS rSrg)BrosPreTrainedModeli  brosc                    [        U[        R                  5      (       ak  UR                  R                  R                  SU R                  R                  S9  UR                  b%  UR                  R                  R                  5         gg[        U[        R                  5      (       ax  UR                  R                  R                  SU R                  R                  S9  UR                  b2  UR                  R                  UR                     R                  5         gg[        U[        R                  5      (       aJ  UR                  R                  R                  5         UR                  R                  R                  S5        gg)zInitialize the weightsr0   )meanstdNg      ?)r	  r   rk   weightdatanormal_r8   initializer_rangeri   zero_r   ry   r   fill_)r7   modules     r+   _init_weights!BrosPreTrainedModel._init_weights  s   fbii(( MM&&CT[[5R5R&S{{&  &&( '--MM&&CT[[5R5R&S!!-""6#5#56<<> .--KK""$MM$$S) .r*   r    N)	r!   r"   r#   r$   r   config_classbase_model_prefixr}  r)   r    r*   r+   rq  rq    s    L*r*   rq  c            "         ^  \ rS rSrSU 4S jjrS rS rS r\              SS\	\
R                     S\	\
R                     S\	\
R                     S	\	\
R                     S
\	\
R                     S\	\
R                     S\	\
R                     S\	\
R                     S\	\
R                     S\	\\
R                        S\	\   S\	\   S\	\   S\	\   S\\\
R                     \4   4S jj5       rSrU =r$ )	BrosModeli  c                    > [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        [        U5      U l        U(       a  [        U5      OSU l
        U R                  5         g)z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
N)r2   r3   r8   rv   r   rf   bbox_embeddingsr0  encoderrT  poolerinit_weights)r7   r8   add_pooling_layerr9   s      r+   r3   BrosModel.__init__  sX    
 	 ,V41&9"6*,=j(4r*   c                 .    U R                   R                  $ rW   r   r   )r7   s    r+   get_input_embeddingsBrosModel.get_input_embeddings  s    ...r*   c                 $    XR                   l        g rW   r  )r7   r   s     r+   set_input_embeddingsBrosModel.set_input_embeddings  s    */'r*   c                     UR                  5        H7  u  p#U R                  R                  U   R                  R	                  U5        M9     g)z
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
class PreTrainedModel
N)itemsr  r4  r  r   )r7   heads_to_pruner4  r   s       r+   _prune_headsBrosModel._prune_heads  s<    
 +002LELLu%//;;EB 3r*   r   r]   r   r   r~   r   r   r   r   r7  r8  r   r9  r:  r<   c                 F   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nU R                   R                  (       a  Ub  UOU R                   R
                  nOSnUb  Ub  [        S5      eUb  UR                  5       nO"Ub  UR                  5       SS nO[        S5      eUc  [        S5      eUu  nnUb  UR                  OUR                  nU
b  U
S   S   R                  S   OSnUc  [        R                  " UUS	9nUcs  [        U R                  S
5      (       a4  U R                  R                  SS2SU24   nUR                  UU5      nUnO$[        R                   " U[        R"                  US9nU R%                  X?U5      nU R                   R                  (       aE  UbB  UR                  5       u  nnnUU4nU	c  [        R                  " UUS	9n	U R'                  U	5      nOSnU R)                  X`R                   R*                  5      nU R                  UUUUUS9nUR                  S   S:X  a  USS2SS2/ SQ4   nX R                   R,                  -  nU R/                  U5      nU R1                  UUUUUUU
UUUUS9nUS   n U R2                  b  U R3                  U 5      OSn!U(       d
  U U!4USS -   $ [5        U U!UR6                  UR8                  UR:                  UR<                  S9$ )aM  
bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'):
    Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values
    (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the
    bounding box.

Examples:

```python
>>> import torch
>>> from transformers import BrosProcessor, BrosModel

>>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

>>> model = BrosModel.from_pretrained("jinho8345/bros-base-uncased")

>>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
>>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
>>> encoding["bbox"] = bbox

>>> outputs = model(**encoding)
>>> last_hidden_states = outputs.last_hidden_state
```NFzDYou cannot specify both input_ids and inputs_embeds at the same timer?   z5You have to specify either input_ids or inputs_embedszYou have to specify bboxr   r>   )r   r   r   )r   r~   r   r   r      )r   r   r>   r   r>   r
   r   r
   )
rc   r   r   r   r   r7  r8  r   r9  r:  r   )rB  pooler_outputr7  r   r   rC  )r8   r   r9  use_return_dictr   r8  r   rB   r   r   r&   onesr   r   r   r   r   r   get_extended_attention_maskinvert_attention_maskget_head_maskr3  
bbox_scaler  r  r  r   r7  r   r   rC  )"r7   r   r]   r   r   r~   r   r   r   r   r7  r8  r   r9  r:  r   r   r   r   r   r   r   extended_attention_maskencoder_batch_sizeencoder_sequence_lengthr5  encoder_hidden_shapeencoder_extended_attention_maskembedding_outputscaled_bboxbbox_position_embeddingsencoder_outputssequence_outputr[  s"                                     r+   rM   BrosModel.forward  sR   R 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B];;!!%.%:	@U@UII ]%>cdd"#..*K&',,.s3KTUU<788!,
J%.%:!!@T@T DSC^!3A!6!<!<Q!?de!"ZZFCN!t(899*.//*H*HKZK*X'3J3Q3QR\^h3i0!A!&[

SY!Z 150P0PQ_nt0u ;;!!&;&G=R=W=W=Y: 7$68O#P %-).4HQW)X&.2.H.HI_.`+.2+ &&y++2O2OP	??%)'#9 + 
 ::b>Q1667D[[333#'#7#7#D ,,12"7#B+/!5# ' 
 *!,8<8OO4UY#]3oab6III;-'+;;)77&11,==
 	
r*   )r  r8   r   r  r  )T)NNNNNNNNNNNNNN)r!   r"   r#   r$   r3   r  r  r  r   r   r&   rP   r   r'   r  r   r   r   rM   r)   rQ   rR   s   @r+   r  r    s    /0C  -1'+1515/3,0048<9==A$(,0/3&*N
ELL)N
 u||$N
 !.	N

 !.N
 u||,N
 ELL)N
  -N
  (5N
 !) 6N
 "$u'8'8"9:N
 D>N
 $D>N
 'tnN
 d^N
  
uU\\"$PP	Q!N
 N
r*   r  c                     ^  \ rS rSrS/rU 4S jr\            SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\
   S\\
   S\\
   S\\\R                     \4   4S jj5       rSrU =r$ )BrosForTokenClassificationi`  r  c                 n  > [         TU ]  U5        UR                  U l        [        U5      U l        [        US5      (       a  UR                  OUR                  n[        R                  " U5      U l
        [        R                  " UR                  UR                  5      U l        U R                  5         g Nclassifier_dropout)r2   r3   
num_labelsr  rr  r   r  r   r   r   r   rk   r   
classifierr  r7   r8   r  r9   s      r+   r3   #BrosForTokenClassification.__init__d  s      ++f%	)09M)N)NF%%TZTnTn 	 zz"45))F$6$68I8IJr*   r   r]   r   bbox_first_token_maskr   r~   r   r   labelsr   r9  r:  r<   c                 H   Ub  UOU R                   R                  nU R                  UUUUUUUU
UUS9
nUS   nU R                  U5      nU R	                  U5      nSnU	b  [        5       nUbJ  UR                  S5      nU" UR                  SU R                  5      U   U	R                  S5      U   5      nO2U" UR                  SU R                  5      U	R                  S5      5      nU(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  S9$ )aP  
bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'):
    Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values
    (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the
    bounding box.
bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

Examples:

```python
>>> import torch
>>> from transformers import BrosProcessor, BrosForTokenClassification

>>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

>>> model = BrosForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

>>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
>>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
>>> encoding["bbox"] = bbox

>>> outputs = model(**encoding)
```N)	r]   r   r   r~   r   r   r   r9  r:  r   r?   r>   r   logitsr   r   )r8   r  rr  r   r  r	   rC   r  r   r   r   )r7   r   r]   r   r  r   r~   r   r   r  r   r9  r:  r   r  r  r   loss_fctr   s                      r+   rM   "BrosForTokenClassification.forwardq  sF   X &1%<k$++B]B]))))%'/!5#  
 "!*,,71')H$0(=(B(B2(F%KKDOO45JKV[[Y[_]rMs  B @&++b/RY,F)-)9TGf$EvE$!//))	
 	
r*   )rr  r  r   r  NNNNNNNNNNNNr!   r"   r#   r$   "_keys_to_ignore_on_load_unexpectedr3   r   r   r&   rP   r  r   r   r   rM   r)   rQ   rR   s   @r+   r  r  `  sJ   *3&  -1'+158<15/3,004)-,0/3&*S
ELL)S
 u||$S
 !.	S

  (5S
 !.S
 u||,S
 ELL)S
  -S
 &S
 $D>S
 'tnS
 d^S
 
uU\\"$99	:S
 S
r*   r  a  
    Bros Model with a token classification head on top (initial_token_layers and subsequent_token_layer on top of the
    hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. The initial_token_classifier is used to
    predict the first token of each entity, and the subsequent_token_classifier is used to predict the subsequent
    tokens within an entity. Compared to BrosForTokenClassification, this model is more robust to serialization errors
    since it predicts next token from one token.
    )custom_introc                      ^  \ rS rSrS/rU 4S jr\             SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\R                     S\\
   S\\
   S\\
   S\\\R                     \4   4S jj5       rSrU =r$ )!BrosSpadeEEForTokenClassificationi  r  c           	      x  > [         TU ]  U5        Xl        UR                  U l        UR                  U l        UR
                  U l        [        U5      U l        [        US5      (       a  UR                  OUR                  n[        R                  " [        R                  " U5      [        R                  " UR
                  UR
                  5      [        R                  " U5      [        R                  " UR
                  UR                  5      5      U l        [#        U5      U l        U R'                  5         g r  )r2   r3   r8   r  r`  r   ra  r  rr  r   r  r   r   
Sequentialr   rk   initial_token_classifierr^  subsequent_token_classifierr  r  s      r+   r3   *BrosSpadeEEForTokenClassification.__init__  s      ++!--$*$6$6!f%	)09M)N)NF%%TZTnTn 	
 )+JJ)*IIf((&*<*<=JJ)*IIf((&*;*;<	)
% ,A+H(r*   r   r]   r   r  r   r~   r   r   initial_token_labelssubsequent_token_labelsr   r9  r:  r<   c                    Ub  UOU R                   R                  nU R                  UUUUUUUUUUS9
nUS   nUR                  SS5      R	                  5       nU R                  U5      R                  SS5      R	                  5       nU R                  X5      R                  S5      nSU-
  nUR                  u  nnUR                  n[        R                  " U[        R                  " US/5      R                  U5      /SS9R                  5       nUR                  USS2SSS24   [        R                   " UR"                  5      R$                  5      n[        R&                  " UUS-   5      R                  U[        R                  S9nUR                  USSS2SS24   [        R                   " UR"                  5      R$                  5      nUR)                  S5      R                  5       nSnU	b  U
b  [+        5       nU	R)                  S5      n	Ub:  UR)                  S5      nU" UR)                  SU R,                  5      U   X   5      nO#U" UR)                  SU R,                  5      U	5      nU
R)                  S5      n
U" UR)                  SUS-   5      U   U
U   5      nUU-   nU(       d  UU4USS -   nUb  U4U-   $ U$ [/        UUUUR0                  UR2                  S	9$ )
a  
bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'):
    Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values
    (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the
    bounding box.
bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.
initial_token_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for the initial token classification.
subsequent_token_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for the subsequent token classification.

Examples:

```python
>>> import torch
>>> from transformers import BrosProcessor, BrosSpadeEEForTokenClassification

>>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

>>> model = BrosSpadeEEForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

>>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
>>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
>>> encoding["bbox"] = bbox

>>> outputs = model(**encoding)
```N
r   r]   r   r   r~   r   r   r   r9  r:  r   r   ri  r   r   r?   r>   )r   r   r   r   r   )r8   r  rr  rq   r   r  r  squeezer   r   r&   rD   r   r   r  masked_fillfinfor   mineyerC   r	   r  r   r   r   )r7   r   r]   r   r  r   r~   r   r   r  r  r   r9  r:  r   last_hidden_statesr   r   inv_attention_maskr   max_seq_lengthr   invalid_token_maskself_token_masksubsequent_token_maskr   r  initial_token_losssubsequent_token_lossr   s                                 r+   rM   )BrosSpadeEEForTokenClassification.forward  s   b &1%<k$++B]B]))))%'/!5#  
 %QZ/99!Q?JJL#<<=OPZZ[\^_`kkm"&"B"BCU"j"r"rst"u /%7%=%="
N#**"YY(:EKKUV<X<[<[\b<c'dklmrrt"9"E"Eq$z*EKK8O8U8U,V,Z,Z#
  ))NNQ4FGJJRX`e`j`jJk"9"E"ED!QJ'5L5R5R)S)W)W#
 !/ 3 3B 7 < < >+0G0S')H $8#<#<R#@ $0(=(B(B2(F%%-(--b$//BCXY(?&"
 &..B.G.GDOO.\^r%s"&=&B&B2&F#$,',,R!1CDEZ['(=>%!
 &(==D*,CDwqr{RF)-)9TGf$EvE!5$;!//))
 	
r*   )ra  rr  r8   r  r`  r  r  )NNNNNNNNNNNNN)r!   r"   r#   r$   r  r3   r   r   r&   rP   r  r   r   r   rM   r)   rQ   rR   s   @r+   r  r    sb    +4&2  -1'+158<15/3,0047;:>,0/3&*s
ELL)s
 u||$s
 !.	s

  (5s
 !.s
 u||,s
 ELL)s
  -s
 'u||4s
 "*%,,!7s
 $D>s
 'tns
 d^s
 
uU\\"O3	4s
 s
r*   r  z
    Bros Model with a token classification head on top (a entity_linker layer on top of the hidden-states output) e.g.
    for Entity-Linking. The entity_linker is used to predict intra-entity links (one entity to another entity).
    c                     ^  \ rS rSrS/rU 4S jr\            SS\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                     S\\R                     S\\
   S\\
   S\\
   S\\\R                     \4   4S jj5       rSrU =r$ )!BrosSpadeELForTokenClassificationid  r  c                 H  > [         TU ]  U5        Xl        UR                  U l        UR                  U l        UR
                  U l        [        U5      U l        [        US5      (       a  UR                  OUR                    [        U5      U l        U R                  5         g r  )r2   r3   r8   r  r`  r   ra  r  rr  r   r  r   r^  entity_linkerr  r[   s     r+   r3   *BrosSpadeELForTokenClassification.__init__m  s      ++!--$*$6$6!f%	&-f6J&K&K	"	"QWQkQk26:r*   r   r]   r   r  r   r~   r   r   r  r   r9  r:  r<   c                 6   Ub  UOU R                   R                  nU R                  UUUUUUUU
UUS9
nUS   nUR                  SS5      R	                  5       nU R                  X5      R                  S5      nSnU	Gba  [        5       nUR                  u  nnUR                  n[        R                  " UUS-   5      R                  U[        R                  S9nUR                  S5      n[        R                  " U) [        R                   " US/[        R                  US9/SS9nUR#                  USS2SSS24   [        R$                  " UR&                  5      R(                  5      nUR#                  USSS2SS24   [        R$                  " UR&                  5      R(                  5      nU" UR                  SUS-   5      U   U	R                  S5      U   5      nU(       d  U4US	S -   nUb  U4U-   $ U$ [+        UUUR,                  UR.                  S
9$ )a^  
bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'):
    Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values
    (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the
    bounding box.
bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

Examples:

```python
>>> import torch
>>> from transformers import BrosProcessor, BrosSpadeELForTokenClassification

>>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

>>> model = BrosSpadeELForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

>>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
>>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
>>> encoding["bbox"] = bbox

>>> outputs = model(**encoding)
```Nr  r   r   r  r?   r   ri  r>   r  )r8   r  rr  rq   r   r  r  r	   r   r   r&   r  r   r  rC   rD   r   r  r  r   r  r   r   r   )r7   r   r]   r   r  r   r~   r   r   r  r   r9  r:  r   r  r  r   r  r   r  r   r  maskr   s                           r+   rM   )BrosSpadeELForTokenClassification.forward{  s   V &1%<k$++B]B]))))%'/!5#  
 %QZ/99!Q?JJL##$6KSSTUV')H)7)=)=&J#**F#ii8JKNNV\didndnNoO(--b1D$)II**KKQuzz&Q %! ''(=aqj(I5;;W]WcWcKdKhKhiF''a
(CU[[QWQ]Q]E^EbEbcFFKKNQ,>?Ev{{SUW[G\]DY,F)-)9TGf$EvE$!//))	
 	
r*   )ra  rr  r8   r  r`  r  r  r  rR   s   @r+   r  r  d  sL    +4&  -1'+158<15/3,004)-,0/3&*]
ELL)]
 u||$]
 !.	]

  (5]
 !.]
 u||,]
 ELL)]
  -]
 &]
 $D>]
 'tn]
 d^]
 
uU\\"$99	:]
 ]
r*   r  )rq  r  r  r  r  )8r%   r   dataclassesr   typingr   r   r   r   r&   torch.utils.checkpointr   torch.nnr	   activationsr   modeling_outputsr   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   configuration_brosr   
get_loggerr!   rF  r   Moduler-   rT   rf   rv   r   r   r   r  r  r  r0  rT  r^  rq  r  r  r  r  __all__r    r*   r+   <module>r     s     ! / /    % ! 
 . l l 9 9 * 
		H	% :k : :>		 *		 & ? ?DG		 GVRYY 3BII 3nryy  W		 WtZ
")) Z
| BII D */ * ** n
# n
 n
b d
!4 d
 d
N P
(; P
P
f o
(; o
o
dr*   