
    eThe              
       N   S SK r S SKrS SKJrJrJrJrJr  S SKr	S SK
r
S SKJr  SSKJr  SSKJr  \" 5       (       a  S SKJr  SSKJr  SSKJr  S	S
KJrJrJr  \(       a  SSKJr  SSKJr  S	SKJr  SSK J!r!   " S S5      r" " S S\"5      r# " S S\#5      r$ " S S\RJ                  5      r& " S S\RJ                  5      r' " S S5      r( " S S5      r) " S S\$5      r* " S S \"5      r+ " S! S"\#5      r,S# r-S$\\.\4   S%\/S&\0S'\\.\4   4S( jr1S$\\.\4   S%\/S'\\.\4   4S) jr2g)*    N)TYPE_CHECKINGAnyDictOptionalTuple   )prune_linear_layer)is_sklearn_available)	roc_curve)Cache)isin_mps_friendly   )LogitsProcessorListMinLengthLogitsProcessorSuppressTokensLogitsProcessor)PreTrainedModel)PreTrainedTokenizerBase)GenerationConfig)deprecate_kwargc                       \ rS rSrSrS\R                  S\\R                  \\R                     4   4S jr
S\R                  S\R                  S\4S jrS	rg
)CandidateGenerator,   z`Abstract base class for all candidate generators that can be applied during assisted generation.	input_idsreturnc                 2    [        U R                   S35      e)a  
Fetches the candidates to be tried for the current input.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

Return:
    `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be
    assessed by the model and, optionally, a `torch.FloatTensor` of shape `(batch_size, candidate_length,
    vocabulary_size)` containing the logits associated to each candidate.
zT is an abstract class. Only classes inheriting this class can call `get_candidates`.NotImplementedError	__class__)selfr   s     c/var/www/auris/envauris/lib/python3.13/site-packages/transformers/generation/candidate_generator.pyget_candidates!CandidateGenerator.get_candidates/   s!     "~~rs
 	
    scoresnum_matchesc                 2    [        U R                   S35      e)  
Updates the candidate generation strategy based on the outcomes.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
    scores (`torch.FloatTensor` of shape `(batch_size, candidate_length, config.vocab_size)`):
        Prediction scores of a language modeling head. These can be logits for each vocabulary when not using
        beam search or log softmax for each vocabulary token when using beam search
    num_matches (`int`):
        The number of matches between the candidate sequences and the model predictions.
z_ is an abstract class. Only classes inheriting this class can call `update_candidate_strategy`.r   r   r   r$   r%   s       r    update_candidate_strategy,CandidateGenerator.update_candidate_strategy@   s%     "~~ + +
 	
r#    N)__name__
__module____qualname____firstlineno____doc__torch
LongTensorr   r   FloatTensorr!   intr)   __static_attributes__r+   r#   r    r   r   ,   sb    j
(8(8 
U5CSCSU]^c^o^oUpCp=q 
"
53C3C 
UM^M^ 
mp 
r#   r   c                      \ rS rSrSr  SS\R                  SSSSS	\S
\\R                     SS4S jjr
S\R                  S\\R                  \\R                     4   4S jrS\R                  S\R                  S\4S jrS\R                  S\\\4   4S jr SS\R                  S\S\S\4S jjrS\R                  S\S\S\4S jrS\S\\R                  \\R                     4   4S jrSrg)AssistedCandidateGeneratorS   a&  
`CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates
candidates through the use of a smaller model. Read the following blog post for more information:
https://huggingface.co/blog/assisted-generation

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
    assistant_model (`PreTrainedModel`):
        The model to be used for generating candidates. This model should be smaller than the main model.
    generation_config (`~generation.GenerationConfig`, *optional*):
        The generation configuration to be used as base parametrization for the generation call.
    logits_processor (`LogitsProcessorList`):
        An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
        used to modify the prediction scores of the language modeling head applied at each generation step.
    model_kwargs (`Dict`):
        The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
        model as well.
    inputs_tensor (`torch.Tensor`, *optional*):
        The model input tensor. In encoder-decoder models, this is the encoder input.
Nr   assistant_modelr   generation_configr   model_kwargsinputs_tensorlogits_processorr   c           	      >   UR                   nUR                  U5      nUb  UR                  U5      nX l        UR                  R                  U l        UR                  R
                  U l        UR                  U R                  R                  l        0 nUR                  5        Hc  u  pU	S;  d  M  [        U
[        R                  5      (       a  U
R                  5       R                  U5      O[        R                  " U
5      X'   Me     SUR                  5       ;   a  UR                  5       (       d  US	 UR                   R"                  (       aF  UR%                  XRR                  R&                  U5      u  p[nUR)                  XXXR                  5      nOSU;   a  US   US'   Xl        UR                   R"                  (       a  SU l        OySU;   al  SU l        U R*                  R/                  S[        R0                  " UR2                  S   S4UR                   [        R4                  S	95      U R*                  S
'   OSU l        Ub  UO	[7        5       U l        [        R                  " U5      U l        SU R                  l        SU R                  l        U R
                  U R                  l        SU R                  l        U R                  R@                  U l!        SU R                  l         S U R                  l"        U R8                   H#  n[        U[F        5      (       d  M  [I        S5      e   S U R                  l%        [M        5       (       aH  U R                  R                  R
                  (       a"  [O        U 5      [P        L a  / U l)        / U l*        g g g g )N)encoder_outputspast_key_valueslogits_to_keepr?   decoder_input_idsr   decoder_attention_maskr   r   )devicedtypeattention_maskTzPassing `MinLengthLogitsProcessor` when using `assisted_generation is disabled. Please pass in `min_length` into `.generate()` instead)+rD   tor9   r:   num_assistant_tokensassistant_confidence_thresholdeos_token_iditems
isinstancer1   Tensordetachcopydeepcopykeys_supports_logits_to_keepconfigis_encoder_decoder_prepare_model_inputsbos_token_id._prepare_encoder_decoder_kwargs_for_generationassistant_kwargsinput_ids_keygetonesshapelongr   r=   return_dict_in_generateoutput_scoresis_assistant
min_lengthmain_model_min_lengthmin_new_tokensr   
ValueErrorcache_implementationr
   typer7   probsmatches)r   r   r9   r:   r;   r<   r=   rD   rX   keyvaluemodel_input_name	processors                r    __init__#AssistedCandidateGenerator.__init__j   s0    !''LL(	$),,V4M  /$3$E$E$Z$Z!.=.O.O.n.n+ ?P>\>\..; &,,.JC@@1;E5<<1P1PELLN%%f-VZVcVcdiVj !% / /4466?g?g?i?i !12 !!44@O@e@e@@MMO_A=M-=  /]]1ACdCd  ,.2>?P2Q./ 0 !!44!4D"22!,D6:6K6K6O6O(

IOOA.29;K;KSXS]S]^7D!!"23 "-D 5E4P 0ViVk!%/@!A9=6/3,@D@c@c=.2+ &*%;%;%F%F",-)04-..I)%=>> M  / 7;3 !""$$66UUT
88DJDL 9 V #r#   r   c                     UR                  U R                  R                  5      nU R                  U5      u  p#US:X  a  US4$ U R	                  U5        U R                  XU5      nU R                  U5      u  pVXV4$ )  
Fetches the candidates to be tried for the current input.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

Return:
    `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be
    assessed by the model and a `torch.FloatTensor` of shape `(batch_size, candidate_length,
    vocabulary_size)` containing the logits associated to each candidate.
r   N)rG   r9   rD   _calculate_new_tokens_update_past_and_masks_prepare_generation_args_generate_candidates)r   r   rc   max_new_tokensgeneration_argscandidate_idscandidate_logitss          r    r!   )AssistedCandidateGenerator.get_candidates   s     LL!5!5!<!<=	)-)C)CI)N&Qd?"##I.77	Sab*.*C*CO*T'..r#   r$   r%   c                    U R                   R                  R                  S;   aI  U[        US   5      S-
  :X  a  U =R                  S-  sl        O[        SU R                  S-
  5      U l        [        5       (       Ga  U R                   R                  R                  (       Ga]  [        U 5      [        L GaI  U R                  R                  S/U-  5        [        U R                  5      [        U R                  5      :  a  U R                  R                  S5        [        U R                  5      [        U R                  5      -
  nUS:  a  U R                  U* S2	 [        U R                  5      S:  a  SS1R                  U R                  5      (       ae  [        U R                  U R                  5      u  pVnSU-
  nUSU-  -   n	[         R"                  " U	5      n
Xz   nXR                   R                  l        gggggg)	r'   >   	heuristicheuristic_transientr   r   g       @g      ?N      )r9   r:   num_assistant_tokens_schedulelenrH   maxr
   rI   rf   r7   rh   extendrg   appendissubsetr   npargmin)r   r   r$   r%   excess_lengthfprtpr
thresholdsfnrcostsoptimal_threshold_indexbest_thresholds               r    r)   4AssistedCandidateGenerator.update_candidate_strategy   s     11OO T
 

 c&)nq00))S0),/T5N5NQT5T,U)
 !""$$66UUUT
88 LLk 124::T\\!22##A&  

Oc$,,.??Mq JJ~/ DJJ!#A(E(E'0tzz'J$*#g a#g +-))E*:'!+!DXf$$66U )F# 9 V #r#   c                     UR                   S   n[        [        U R                  5      U R                  R
                  U-
  S-
  5      n[        [        X0R                  U-
  5      S5      nXC4$ )zCCalculate the minimum and maximum number of new tokens to generate.r   r   )r\   minr4   rH   r:   
max_lengthr   rb   )r   r   new_cur_lenru   rc   s        r    rq   0AssistedCandidateGenerator._calculate_new_tokens  se    oob)S!:!:;T=S=S=^=^al=lop=pqS1K1Kk1YZ\]^--r#   remove_from_pkvnum_added_tokensc                    U R                   R                  SS5      SLnU(       a  UR                  S   S-
  U-
  n[        U R                  U R                   S   XS-
  5      U R                   S'   [        U R                   UR                  S   U R                  R                  R                  5      U l         [        U R                   UR                  S   5      U l         U$ )zLUpdate past key values and attention masks for subsequent generation rounds.r@   Nr   r   )	rX   rZ   r\   _crop_past_key_valuesr9   _prepare_attention_maskrS   rT   _prepare_token_type_ids)r   r   r   r   has_past_key_valuesnew_cache_sizes         r    rr   1AssistedCandidateGenerator._update_past_and_masks#  s     #33778I4PX\\&__R014FN7L$$d&;&;<M&NP^Pq8D!!"34 %<%%yr':D<P<P<W<W<j<j%D! %<D<Q<QS\SbSbceSf$gD!""r#   rc   ru   c           
      V    U R                   USUSUSU R                  SU R                  0$ )z*Prepare arguments for the generation call.rc   ru   r:   r=   )rY   r:   r=   )r   r   rc   ru   s       r    rs   3AssistedCandidateGenerator._prepare_generation_args4  s9     	nn!7!7 5 5
 	
r#   rv   c                    U R                   R                  " S0 UDU R                  D6nUR                  U R                  S'   [	        5       (       a  U R                   R
                  R                  (       a  [        U 5      [        L a  [        R                  " UR                  SS9n[        R                  " USS9nUR                  S[        UR                  5      * S24   nU[        [        U5      5      U4   nU R                   R#                  UR%                  5       5        [        R&                  " UR                  SS9nUR                  nX4$ )z7Generate candidate sequences using the assistant model.r@   r   dimr   Nr   r+   )r9   generaterX   r@   r
   r:   rI   rf   r7   r1   catr$   softmax	sequencesr   rangerg   r   toliststack)	r   rv   assistant_outputscores_tensorscores_softmaxidsprx   rw   s	            r    rt   /AssistedCandidateGenerator._generate_candidates>  s   //88d?ddNcNcd3C3S3S/0 ""$$66UUT
88!II&6&=&=1EM"]]=bAN",,R#6F6M6M2N1N1P-PQCuSX34AJJahhj) ;;'7'>'>AF(22..r#   )
rI   rX   r9   r:   rY   r=   rb   rh   rH   rg   NN)r   r   )r,   r-   r.   r/   r0   r1   r2   r   r   rM   rm   r   r3   r!   r4   r)   rq   boolrr   rs   rt   r5   r+   r#   r    r7   r7   S   st   8 1526]##] +] .	]
 ]  -] 0]~/(8(8 /U5CSCSU]^c^o^oUpCp=q /28g53C3C 8gUM^M^ 8gmp 8gt.u/?/? .E#s(O . ^_#))#<?#WZ#	#"
%2B2B 
TW 
il 
qu 
/D /U5CSCSU]^c^o^oUpCp=q /r#   r7   c                     ^  \ rS rSrSr  SS\R                  SSSSSSS	S
S\S\\R                     SS4U 4S jjjr
\S 5       r\S 5       r\S 5       rS rS\R                  S\\R                  \\R"                     4   4S jrS\R                  S\\R                  \4   4S jrS\R                  S\R                  S\R                  S\R                  4S jrSrU =r$ )-AssistedCandidateGeneratorDifferentTokenizersiQ  a  
`CandidateGenerator` class to be used for Universal Assisted Generation (UAD): assisted generation with different tokenizers
for the assistant and main models. This class generates candidates through the use of a smaller
model.

The main model input tokens are re-encoded into assistant model tokens, then candidate tokens are generated in the assistant encoding, which are
in turn re-encoded into main model candidate tokens. Validation then proceeds as explained above.
The re-encoding steps involve decoding token ids into text and then encoding the text using a different tokenizer.
Since re-encoding the tokens may result in tokenization discrepancies, UAD finds the longest common subsequence between the source and target encodings,
to ensure the new tokens include the correct prompt suffix.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
    assistant_model (`PreTrainedModel`):
        The model to be used for generating candidates. This model should be smaller than the main model.
    target_tokenizer (`PreTrainedTokenizerBase`):
        The tokenizer used for the target model.
    assistant_tokenizer (`PreTrainedTokenizerBase`):
        The tokenizer used for the assistant model.
    generation_config (`~generation.GenerationConfig`, *optional*):
        The generation configuration to be used as base parametrization for the generation call.
    logits_processor (`LogitsProcessorList`):
        An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
        used to modify the prediction scores of the language modeling head applied at each generation step.
    model_kwargs (`Dict`):
        The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
        model as well.
    inputs_tensor (`torch.Tensor`, *optional*):
        The model input tensor. In encoder-decoder models, this is the encoder input.
r   r9   r   target_tokenizerr   assistant_tokenizerr:   r   r;   r<   r=   r   c	                    > [         T	U ]  XXVXx5        X0l        X@l        S U l        S U l        UR                  R                  U l        UR                  R                  U l        g N)	superrm   r   r   prev_target_ids_lenprev_assistant_idsr:   target_lookbehindassistant_lookbehind)
r   r   r9   r   r   r:   r;   r<   r=   r   s
            r    rm   6AssistedCandidateGeneratorDifferentTokenizers.__init__r  s\     	5FVcv 0#6 26 "&!0!B!B!T!T$3$E$E$Z$Z!r#   c                    [        5       n0 nU H  n[        R                  " U5      n[        UR	                  5       5      nXb;   a  M9  UR                  U5        SnUS-  nUS   U R                  S   :  a  US   U R                  S   :  at  [        UR	                  5       5      nUR                  U5        XS   US   4   S:X  a  US-  nUS-  nOO.US   U R                  S   :  a  US   U R                  S   :  a  Mt  XsU'   M     U$ )a  
Calculates the length of the longest diagonal sequence in a given matrix.
Args:
    input_matrix (torch.Tensor): The input matrix.
    nonzero_idx (torch.Tensor): The indices of the non-zero elements in the matrix.
Returns:
    dict: A dictionary where the keys are the indices of the non-zero elements and the values are the lengths of the longest diagonal sequences starting from those indices.
r   r   )setr1   clonetupler   addr\   )input_matrixnonzero_idxvisiteddiagsidx	start_idxtuple_start_idxcur_diag_lens           r    _get_longest_diag_dictDAssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_dict  s    %CC(I#I$4$4$67O)KK(LNIA,!3!3A!669Q<,J\J\]^J_;_"'	(8(8(:";O,!il :;q@ A%LNI A,!3!3A!669Q<,J\J\]^J_;_ &#J) * r#   c                     [         R                  X R                  5       5      n[        UR	                  5       5      n[        UR                  5       5      n[        R                  " U5      nX4   nX$   nXV4$ )z
Returns the start index and length of the longest diagonal in the given input.
Args:
    input_matrix (numpy.ndarray): The input matrix.
Returns:
    tuple: A tuple containing the start index and length of the longest diagonal.
)r   r   nonzerolistvaluesrQ   r   argmax)r   r   diags_values
diags_keys	best_diagdiag_start_indexdiag_start_lengths          r    _get_longest_diag_indexEAssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_index  sk     >TT..0
 ELLN+%**,'
IIl+	%0(322r#   c                    UR                   U :H  n[        R                  " U5      (       d  [        R                  " U5      nUR	                  [
        5      nUR                  5       R                  5       (       d  g[        R                  U5      u  pEUS   U-   nUS   U-   nU R                  S   U-
  R                  5       nUSS2Xh-   S24   n	USS2XfU-   24   n
XU
4$ )ay  
Input:
    prompt: 2D array of shape (batch_size, prompt_length), represents the original prompt tokens
    prompt_plus_new_tokens: 2D array of shape (batch_size, prompt_length), represents the suffix of the original prompt, with additional new tokens.
Output:
    discrepancy_length: int, represents the number of tokens that need to be replaced from prompt
    new_tokens_only: 2D array of shape (batch_size, new_token_length), represents the new tokens that are not in prompt
    discrepancy_only: 2D array of shape (batch_size, discrepancy_length), represents the new tokens that are in prompt but not in prompt_plus_new_tokens
)NNNr   r   N)Tr1   	is_tensortensorrG   r4   anyitemr   r   r\   )promptprompt_plus_new_tokenscompare_matcompare_mat_intlongest_locationlongest_diag_lengthnew_token_start_indexdiscrepancy_with_olddiscrepancy_lengthnew_tokens_onlydiscrepancy_onlys              r    _get_tokens_diag>AssistedCandidateGeneratorDifferentTokenizers._get_tokens_diag  s     -..&8{++,,{3K%..-""$))++#0]0u0u1
- !1 36I I/25HH$ll1o0DDJJL04I4^4`1`a1$?Q'QQQ
 "4DDDr#   c                 p    UR                  USSS9nU" USSS9S   nUR                  UR                  5      $ )z
Convert token IDs from one tokenizer to another.
Args:
    input_ids: The input token IDs.
    source_tokenizer: The source tokenizer.
    destination_tokenizer: The destination tokenizer.
Returns:
    The converted token IDs.
Tskip_special_tokensclean_up_tokenization_spacesptadd_special_tokensreturn_tensorsr   )batch_decoderG   rD   )r   r   source_tokenizerdestination_tokenizertextdest_idss         r    &convert_source_tokens_to_target_tokensTAssistedCandidateGeneratorDifferentTokenizers.convert_source_tokens_to_target_tokens  sF      ,,YDos,t($W[\]hi{{9++,,r#   r   c                    [        U R                  5      nUS:X  a  US4$ UR                  U R                  R                  5      nSnU R                  U5      u  pCX@l        [        [        X R                  UR                  S   -
  5      S5      nU R                  XC5        U R                  XEU5      nU R                  R                  SS5        U R                  R                  " S0 UDU R                  D6nU R!                  XR"                  U5      nUR                  S   U l        UR&                  U R                  S'   UR"                  U l        U R$                  UR                  S   :  a  US4$ US4$ )rp   r   Nr   rF   r   r@   r+   )r4   rH   rG   r9   rD   _prepare_assistant_input_idsr   r   r   rb   r\   rr   rs   rX   popr   _process_assistant_outputsr   r   r@   )	r   r   ru   r   assistant_input_idsrc   rv   r   new_target_idss	            r    r!   <AssistedCandidateGeneratorDifferentTokenizers.get_candidates  sh    T667Qd?"LL!5!5!<!<=	/3/P/PQZ/[,"5S1K1KNaNgNghjNk1klnop##$7I778K]kl!!"2D9//88d?ddNcNcd88D^D^`st $-??1#5 3C3S3S/0"2"<"<##~';';A'>>d?"t##r#   c                 d   U R                   U R                  S.nSnU R                  Gb^  U R                  U R                  :  GaC  U R                  U R                  -
  nU R
                  " USS2US24   40 UD6nUR                  S   nU R                  SS2U* S24   nU R                  Xu5      u  pn
U R                  nU	b  US:  az  U
R                  S   S:  ag  XR                  S   :X  a  XSS2U* S24'   OGXR                  S   :  a5  XR                  S   -
  nUSS2SU* 24   nXSS2U
R                  S   * S24'   UnU	R                  S   S:  a  [        R                  " X/SS9nX4$ [        R                  " X/SS9n X4$ U R
                  " U40 UD6nUR                  S   U l        X4$ )zIConverts target input IDs to assistant input IDs, handling discrepancies.r   r   r   Nr   r   r   )
r   r   r   r   r   r   r\   r   r1   r   )r   r   convert_kwargsr   start_index_in_target_windownew_assistant_idsprompt_use_length
prompt_user   r   r   r   discrepancy_length_diffs                r    r   JAssistedCandidateGeneratorDifferentTokenizers._prepare_assistant_input_ids  s    !% 5 5%)%=%=
 "".43K3KdNdNd3d+/+C+CdF\F\+\( $ K K!9::;!?M! !2 7 7 :005F4F4G1GHJDHDYDYEA1A #'"9"9*%).>.D.DQ.G!.K)-C-CA-FFGWA0B/B/C,CD+.D.DQ.GG2DG]G]^_G`2`/.A!E^G^F^E^B^._+O_A0@0F0Fq0I/I/K,KL&8O"((+a/*/))5H4Z`b*c' #33 ',ii1D0X^`&a#
 #33 #'"M"Mi"j[i"j'0q'9D$"33r#   assistant_sequencesr   c                    U R                   R                  S   nX@R                  -
  nU R                  USS2US24   U R                  U R
                  S9nUR                  S   nUSS2U* S24   nU R                  X5      u  pn	UnU
b*  U
R                  S   S:  a  [        R                  " X/SS9nO[        R                  " X/SS9n[        U R                  S5      (       a  USS2SU R                  R                  24   nU$ )z7Processes assistant outputs to obtain target input IDs.r   Nr  r   r   r   r   )r   r\   r   r   r   r   r   r1   r   hasattrr:   r   )r   r   r  r   num_prev_assistantstart_assistant_look_indexnew_target_ids_from_windowtarget_prompt_use_lengthtarget_prompt_use_target_new_tokens_onlyr  s               r    r   HAssistedCandidateGeneratorDifferentTokenizers._process_assistant_outputsJ  s'    "44::1=%7:S:S%S"%)%P%P#=#> >?!55"&"7"7 &Q &
"
 $>#C#CA#F %a*B)B)C&CD'+'<'<=N'k$1"!-%++A.2!&N+SY[!\ #YY'SY[\N4))<88+A/R1G1G1R1R/R,RSNr#   )r   r   r   r   r   r   r   )r,   r-   r.   r/   r0   r1   r2   r   r   rM   rm   staticmethodr   r   r   r   r   r3   r!   r4   r   r   r5   __classcell__r   s   @r    r   r   Q  sl   P 1526[##[ +[ 4	[
 7[ .[ [  -[ 0[ [( ! !F 3 3& E E@-&($(8(8 ($U5CSCSU]^c^o^oUpCp=q ($T,4e6F6F ,45QVQaQacfQfKg ,4\))@E@P@Pglgwgw			 r#   r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )_PruneReindexingLMHeadik  aP  
A class to prune and reindex the language model head.

This class prunes the language model head to only include the specified token IDs and reindexes the logits
to map back to the original vocabulary.

Args:
    original_lm_head (nn.Module): The original language model head.
    token_ids (list[int]): The list of token IDs to keep.
c                    > [         TU ]  5         [        X5      R                  UR                  R
                  5      U l        g r   )r   rm   r	   rG   weightrE   pruned_lm_head)r   original_lm_headassistant_overlap_token_idsr   s      r    rm   _PruneReindexingLMHead.__init__w  s6    01A_bb##))
r#   c                 (    U R                  U5      nU$ r   r  )r   hidden_statespruned_logitss      r    forward_PruneReindexingLMHead.forward}  s    ++M:r#   r#  )	r,   r-   r.   r/   r0   rm   r&  r5   r  r  s   @r    r  r  k  s    	
 r#   r  c                   ~   ^  \ rS rSrS\R
                  4U 4S jjrS\R                  S\R                  4S jr
SrU =r$ )_MapInputEmbeddingi  original_embeddingc                 j   > [         TU ]  5         Xl        UR                  U l        X l        SU l        g)a$  
Wraps an existing embedding layer and remaps token IDs before lookup.

Args:
    original_embedding (nn.Embedding): Pre-trained or existing embedding layer.
    assistant_overlap_token_ids (dict): Mapping from original token IDs to new token IDs.
                  Example: {old_id: new_id}
FN)r   rm   r*  r  r   map)r   r*  r   r   s      r    rm   _MapInputEmbedding.__init__  s0     	"4(//+F(r#   r   r   c                     U R                   (       a1  U R                  US      R                  S5      R                  S5      nO	SU l         UnU R                  U5      $ )z
Args:
    input_ids (torch.LongTensor): Tensor of token IDs (batch_size, seq_len).

Returns:
    torch.FloatTensor: Corresponding input embeddings.
)r   r   r   T)r,  r   	unsqueezer*  )r   r   my_input_idss      r    r&  _MapInputEmbedding.forward  sU     88;;Ie<LMWWXYZddefgLDH$L&&|44r#   )r   r,  r*  r  )r,   r-   r.   r/   nn	Embeddingrm   r1   r2   r3   r&  r5   r  r  s   @r    r)  r)    s7    2<< 5!1!1 5e6G6G 5 5r#   r)  c                      \ rS rSr% Sr\" S5      * r\\S'   Sr\	\S'   \
" SSS	9   SSSSSS\	S\S\S   S\4S jj5       rS rS rS\\	   4S jrS\R(                  S\R(                  4S jrS\R,                  S\R,                  4S jrSrg
)AssistantToTargetTranslatori  a  
Translates token ids and logits between assistant and target model vocabularies. This class is used to handle
vocabulary mismatches when using different tokenizers for the assistant and target models in speculative decoding,
as introduced in the paper "Lossless Speculative Decoding Algorithms for Heterogeneous Vocabularies"
(https://www.arxiv.org/abs/2502.05202).
It maintains mappings between the two vocabularies and handles token/logit conversion.

Args:
    target_tokenizer (`PreTrainedTokenizerBase`):
        The tokenizer used by the target (main) model.
    assistant_tokenizer (`PreTrainedTokenizerBase`):
        The tokenizer used by the assistant model.
    target_vocab_size (`int`):
        The size of the target model's vocabulary. If not provided, will be inferred from the target tokenizer.
    assistant_model_device (str, optional): The device on which the assistant model is loaded.
            Defaults to "cpu".
    assistant_model_device (`str`, defaults to "cpu"): The device where the assistant model is located. Used for placing tensors.
    assistant_model (Optional[PreTrainedModel], optional): The assistant model to be used. Defaults to None for backward compatibility.
    assistant_prune_lm_head (bool): Whether to prune the assistant model's language model
        head to match the target vocabulary. This is only applicable if `assistant_model` is provided.
        Defaults to False for backward compatibility.
InfFILTER_VALUEr   SUPPRESS_TOKEN_IDassistant_model_device4.53versionNr   r   r   target_vocab_sizer9   r   assistant_prune_lm_headc                 :   Xl         X l        Uc  UOUR                  U l        X0l        U R                  5       u  U l        U l        U R                  5       U l	        S U l
        U=(       a    US LU l        [        U R                  5      S:  Ga  U R                  (       a  [        R                  " [        U R                  R!                  5       5      [        R"                  U R                  S9U l        UR'                  5       n[)        XpR$                  5      nAUR+                  U5        UR-                  5       n	[/        XR$                  5      n
A	UR1                  U
5        Xl        g [5        [7        U R                  5       U R                  5      /5      U l
        g g )Nr   )rE   rD   )_target_tokenizer_assistant_tokenizerrD   _assistant_model_devicer=  "_get_assistant_to_target_input_ids_assistant_to_target_input_idstarget_to_assistant_input_ids_get_suppress_input_ids_suppress_input_idslogits_processorsr>  r   r1   r   r   r   r]   r   get_output_embeddingsr  set_output_embeddingsget_input_embeddingsr)  set_input_embeddingsmap_input_embeddingsr   r   )r   r   r   r=  r9  r9   r>  r  r  original_input_embeddingsrM  s              r    rm   $AssistantToTargetTranslator.__init__  sk    =M?R!&5&="?CYCY 	$ '8335 	P+T-O /3.J.J.L @D'>'^?Z^C^$t''(1,++38<<;;BBDE**7740
 $3#H#H#J !78HJjJj!k$55nE,;,P,P,R)'9:SUuUu'v$-445IJ,@))<243O3O3QSWSoSopq*&' -r#   c                 J    U R                   (       a  SU R                  l        gg)a[  
Disables the mapping of input ids despite the assistant pruning for the language model head being enabled.

This method is required for the first forward pass of `_MapInputEmbedding` where input ids are already in the assistant vocabulary space. By disabling the mapping, it ensures that the input ids are processed correctly without remapping.

FN)r>  rM  r,  r   s    r    unmap_input_ids+AssistantToTargetTranslator.unmap_input_ids  s      '',1D%%) (r#   c           	      X   U R                   R                  5       nU R                  R                  5       nSnU R                  USS9S   n[        U5      S:  a  U R                   R	                  U5      S   S   nU R                  USS9S   n[        U5      S:  as  U R                  R	                  U5      S   S   nXW:w  aM  UR                  5        VV	s0 s H0  u  pUR                  U5      (       a  UR                  XuS5      OUU	_M2     nnn	[        UR                  5       5      n
[        R                  " U
S-   4U R                  [        S9n0 nUR                  5        H#  u  pUR                  U5      nUc  M  XU'   XU'   M%     UR                  U R                   5      U4$ s  sn	nf )N F)r   r   r   r   rE   )r@  	get_vocabrA  r   convert_ids_to_tokensrK   
startswithreplacer   r   r1   fullr8  r4   rZ   rG   rB  )r   target_vocabassistant_vocab	space_strtarget_space_idstarget_space_signassistant_space_idsassistant_space_signtokr   max_assistant_indexassistant_to_target_input_idsrE  assistant_id	target_ids                  r    rC  >AssistantToTargetTranslator._get_assistant_to_target_input_ids  s   --77933==?	11)PU1VWbc 1$ $ 6 6 L LM] ^_` abc d"&";";IZ_";"`al"m&'!+'+'@'@'V'VWj'klm'nop'q$$< )8(=(=(?' )@HC  #~~.BCC  KK(<QRS!$ )@ $ ' "/"8"8":;(-

4G!4K3MtOeOemp(q%8:%!0!6!6!8C$((-I$>Gl;;Gi8	 "9
 -//0L0LMOlll#'s   7F&r   c                 b    [         R                  " U R                  U R                  :H  5      S   $ )zP
Get the input ids that are in the assistant vocab but not in the target vocab.
r   )r1   whererD  r8  rQ  s    r    rF  3AssistantToTargetTranslator._get_suppress_input_ids  s*     {{4>>$BXBXXYZ[\\r#   assistant_candidate_idsc                 
   [        US   5      UR                  S   -
  nUS:X  a  U$ USU* S24   nU R                  (       a  U R                  U   nU R                  U   n[
        R                  " X&R                  S5      4SS9$ )a3  
Return the target candidate ids that correspond to the assistant candidate ids.
Note that we have already the target ids for the prompt and we only need to find the target ids for the new tokens.
Moreover, assistant ids of the original prompt does not necessarily appear in _assistant_to_target_input_ids.
r   r   Nr   )r   r\   r>  r   rD  r1   r   r/  )r   r   target_input_idsrl  num_new_tokenslast_candidate_idstransformed_slices          r    get_target_ids*AssistantToTargetTranslator.get_target_ids  s     4Q78;N;T;TUV;WWQ## "9^O<L9L!M++%)%E%EFX%Y" $ C CDV W99.0K0KA0NOUVWWr#   assistant_logitsc                 n   / UR                   SS QU R                  P7n[        R                  " X R                  U R
                  S9nU R                  U R                  :g  nU R                  U   nU R                  (       a  XSU4'   U$ USSU R                  R                   S   24   nUSU4   USU4'   U$ )zC
Return the target logits that correspond to the assistant logits.
Nr   rD   .r   )	r\   r=  r1   r[  r7  rB  rD  r8  r>  )r   rt  target_shapetarget_logitsassistant_indices_masktarget_logits_supported_indicesvalid_assistant_logitss          r    get_target_logits-AssistantToTargetTranslator.get_target_logits4  s    
 )_*:*@*@"*E(^tG]G](^+0::++D4P4P,
 "&!D!DH^H^!^*.*M*MNd*e'''BR#>>?  &6c;iT=`=`=f=fgh=i;i6i%j"BXY\^tYtBuM#>>?r#   )rB  rD  rA  rG  r@  r   r>  rH  rM  rE  r=  cpuNF)r,   r-   r.   r/   r0   floatr7  __annotations__r8  r4   r   strr   r   rm   rR  rC  r   rF  r1   r2   rr  r3   r|  r5   r+   r#   r    r5  r5    s    . !<-L%'s-v> ',7;(-*3* 7* 	*
 !$* ""34* "&* ?*X2!mF]c ]XNSN^N^X			X*%2C2C HYHY r#   r5  c                       \ rS rSrSr\R                  " 5       r\\	" SSS9   SSSS	SS
\
S\S\S   S\S\4S jj5       5       r\S 5       rSrg)AssistantVocabTranslatorCacheiJ  z
Cache for `AssistantToTargetTranslator` instances. The instances are computed at
pre-processing time, and this cache allows us to avoid recomputing them.
r9  r:  r;  Nr   r   r   r=  r9   r   r>  r   c                     U R                   R                  U5      nUc#  [        R                  " 5       nXpR                   U'   UR                  U5      nUc  [	        UUUUUU5      nXU'   U$ r   )_cacherZ   weakrefWeakKeyDictionaryr5  )	clsr   r   r=  r9  r9   r>  assistant_dictmappings	            r    get_translator,AssistantVocabTranslatorCache.get_translatorR  sz     (89!$668N+9JJ'( $$%89?1 #!&'G 3:./r#   c                    U R                    Vs/ s H	  ob  M  UPM     nnU H  nU R                   U	 M     U R                   R                  5        H$  nU Vs/ s H	  ob  M  UPM     nnU H  nX1	 M     M&     gs  snf s  snf )z
Clean up dead references in the cache.
This removes entries where either the target_tokenizer or assistant_tokenizer
has been garbage collected.
N)r  r   )r  ri   	dead_keysr  s       r    cleanup%AssistantVocabTranslatorCache.cleanupp  sy     %(JJ>JSSJ	>C

3  "jj//1N(6FIF "' ! 2 ? Gs   A:A:A?"A?r+   r~  )r,   r-   r.   r/   r0   r  r  r  classmethodr   r4   r  r   r   r5  r  r  r5   r+   r#   r    r  r  J  s    
 &&(F-v> ',7;(-3 7 	
 !$ ""34 "& 
% ? 8 ( (r#   r  c                   V  ^  \ rS rSrSr  SS\R                  SSSSSSS	S
S\S\S\	\R                     SS4U 4S jjjrS\R                  S\\R                  \	\R                     4   4S jrSS\R                  S\S\4U 4S jjjrS\R                  S\R                  4S jrSrU =r$ )%UniversalSpeculativeDecodingGeneratori  z
`CandidateGenerator` class to be used for Universal Speculative Decoding (USD): speculative decoding with different tokenizers
for the assistant and main models. This class generates candidates through the use of a smaller model.
r   r9   r   r   r   r   r:   r   r;   atm_translatorr<   r=   r   c
           
      Z   > Xpl         [        T
U ]	  UUUUUUUU	5        SU l        S U l        g )Nr   )_atm_translatorr   rm   _target_seq_len_with_candidates_prev_assistant_ids)r   r   r9   r   r   r:   r;   r  r<   r=   r   s             r    rm   .UniversalSpeculativeDecodingGenerator.__init__  sB      .		
 56,?C r#   r   c                 X   UR                  U R                  R                  5      nU R                  U5      u  p4U R	                  U5      u  pVUS:X  a  US4$ U R                  X4S9  U R                  X5U5      nSUS   l        SUS   l        U R                  R                  b  U R                  R                  US'   U R                  U5      u  U l        nU R                  R                  X2U R                  5      n	U	R                  S   U l        U R                  R!                  U5      n
X4$ )z[
Simplified version of get_candidates that uses the translator cache for token conversion.
r   Nr   Tr:   r=   r   )rG   r9   rD   r   rq   rr   rs   r_   r^   r  rH  rt   r  rr  r\   r  r|  )r   r   rn  r   r   rc   ru   rv   assistant_candidate_logitstarget_candidate_idstarget_candidate_logitss              r    r!   4UniversalSpeculativeDecodingGenerator.get_candidates  s?    %<<(<(<(C(CD040Q0QRb0c-)-)C)CDT)U&Qd?"##$7#[778K]kl >B+,:GK+,D 11=262F2F2X2XO./?C?X?XYh?i< "<  $33BB43K3K 
 0D/I/I"/M,"&"6"6"H"HIc"d#<<r#   r   r   c                    > U R                   cG  [        U R                  UR                  S   U R                  R
                  R                  5      U l        [        TU ]!  XS9$ )Nr   r  )	r  r   rX   r\   r9   rS   rT   r   rr   )r   r   r   r   s      r    rr   <UniversalSpeculativeDecodingGenerator._update_past_and_masks  s_    ##+ %<%%':'@'@'DdFZFZFaFaFtFt%D! w-.A-eer#   rn  c                 J   UR                   S   nU R                  S:X  a  UnOSnUSS2U* S24   nSnU R                  S:  a6  U R                  R                  R	                  US   R                  5       5      nUcS  U R                  R                  USSS9nU R                  USSS	9S
   R                  U R                  R                  5      nO+[        R                  " U//U R                  R                  S9nU R                  c  UnOTU R                  S-   U-
  nUS:  a  U R                  SS2SU* 24   U l        [        R                  " U R                  U/SS9nUR                  [        R                   S9nU R                  R#                  5         U[%        US   5      4$ )z=
Simplified token conversion that only processes new tokens.
r   r   r   NTr   Fr   r   r   rv  r   rV  )r\   r  r  rE  rZ   r   r   r   r   rG   r9   rD   r1   r   r  r   r]   rR  r   )	r   rn  target_seq_lennew_token_counttarget_new_idsassistant_new_idstarget_new_textr   tokens_to_removes	            r    r   BUniversalSpeculativeDecodingGenerator._prepare_assistant_input_ids  s   
 *//3//14,OO)!o-=->*>? !//!3 $ 4 4 R R V VWefgWhWmWmWo p$"33@@DW[ A O !% 8 8E$ !9 !!2d2299:  !&/@.A-B4K_K_KfKf g ##+"3#CCaG.X!#+/+C+CAGYIYHYGYDY+Z("'))T-E-EGX,Y_a"b1445::4F,,."C(9!(<$===r#   )r  r  r  rX   r   )r   )r,   r-   r.   r/   r0   r1   r2   r   r5  r   rM   rm   r   r3   r!   r4   r   rr   r   r5   r  r  s   @r    r  r    s    1526D##D +D 4	D
 7D .D D 4D  -D 0D D8=(8(8 =U5CSCSU]^c^o^oUpCp=q =@f%:J:J f^a fjn f f&>U=M=M &>RWRbRb &> &>r#   r  c            	           \ rS rSrSr    SS\\R                     S\S\\   S\4S jjr	S	\R                  S
\\R                  \\R                     4   4S jrS	\R                  S\R                  S\4S jrSrg)PromptLookupCandidateGeneratori  a  
`CandidateGenerator` class to be used for prompt lookup generation. This class generates candidates by looking up
likely continuations in the provided prompt (input_ids) itself.
Read the following blog post for more information: https://github.com/apoorvumang/prompt-lookup-decoding

Args:
    max_matching_ngram_size (`int`):
        The maximum ngram size to be considered for matching in the prompt
    num_output_tokens (`int`):
        The number of tokens to be output as candidate tokens.
    max_length (`int`):
        The number of total maximum tokens that can be generated. For decoder-only models that includes the prompt length.
        Defaults to 20, which is the max length used as default in generation config.
NrJ   num_output_tokensmax_matching_ngram_sizer   c                     X l         U(       a  UOSU l        X@l        Xl        U R                  S::  d  U R                   S::  a  [	        S5      eg )Nr   r   z4Invalid max_matching_ngram_size or num_output_tokens)r  r  r   rJ   rd   )r   rJ   r  r  r   s        r    rm   'PromptLookupCandidateGenerator.__init__  sP     "3BY'>_`$$(''1,0F0F!0KSTT 1Lr#   r   r   c                    UR                  S5      nU R                  US-   :X  a  US4$ SnSn[        [        U R                  US-
  5      SS5       H  nUR                  SUSS9nUSU* S24   nXg:H  R                  SS9nUR                  S	S
9S   n	U	 H  n
X-   nXR                  -   n[        XU R                  5      nX:  d  M2  USX24   nS	n[        X0R                  5      n[        R                  " U5      nUR                  5       S:  a  US   R                  5       nUSU n  O   U(       d  M    O   Ub  [        U5      S:X  a  US4$ UR                  S5      n[        R                   " X4SS9nUS4$ )al  
Fetches the candidates to be tried for the current input.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

Return:
    `torch.LongTensor` of shape `(num_candidates, candidate_length)`: The candidate sequences to be tried.
r   NFr   r   )	dimensionsizestepr   r   T)as_tuple)r  r   r   r   r  unfoldallr   r  r   rJ   r1   numelr   r   r/  r   )r   r   input_length
chosen_idsmatch_found
ngram_sizewindowsngram_tensorrh   match_indicesr   r   end_idxmaskmatch_indices_eosfirst_eos_indexcandidate_input_idss                    r    r!   -PromptLookupCandidateGenerator.get_candidates  s    !~~a( ??lQ..d?"
D$@$@,QRBR SUVXZ[J&&!&LG %Q_5L .333:G $OOTO:1=M %,	#&<&<<gT__E&!*1i.?+?!@J"&K
 -Z9J9JKD(-d(;%(..014*;A*>*C*C*E%/0@%A
# %$ {C \F ZA!5d?"  ))!,
#ii(?QG"D((r#   r$   r%   c                     g)r'   Nr+   r(   s       r    r)   8PromptLookupCandidateGenerator.update_candidate_strategyV  s     	r#   )rJ   r   r  r  )N
   N   )r,   r-   r.   r/   r0   r   r1   rM   r4   rm   r2   r   r3   r!   r)   r5   r+   r#   r    r  r    s    " 04!#15Uu||,U U "*#	U
 U>)(8(8 >)U5CSCSU]^c^o^oUpCp=q >)@53C3C UM^M^ mp r#   r  c                      ^  \ rS rSrSr  SS\R                  SSSSS\S	\\R                     S
S4U 4S jjjr
S\R                  S\\R                  \\R                     4   4U 4S jjrSrU =r$ )EarlyExitCandidateGeneratorig  aI  
`CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates
candidates through the use of **the model itself**, exiting early. Can only be used with models that support early
exit, e.g., `facebook/layerskip-llama3.2-1B`.

Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
    assistant_model (`PreTrainedModel`):
        The original model. This model must support early exit (i.e. is trained to compute logits in earlier
        layers).
    generation_config (`~generation.GenerationConfig`, *optional*):
        The generation configuration to be used as base parametrization for the generation call.
    logits_processor (`LogitsProcessorList`):
        An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
        used to modify the prediction scores of the language modeling head applied at each generation step.
    model_kwargs (`Dict`):
        The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
        model as well.
    inputs_tensor (`torch.Tensor`, *optional*):
        The model input tensor. In encoder-decoder models, this is the encoder input.
r   r9   r   r:   r   r;   r<   r=   r   c           	         > [         TU ]  UUUUUUS9  U R                  R                  U l        S U R                  l        g )N)r   r9   r:   r;   r<   r=   )r   rm   r:   assistant_early_exit)r   r   r9   r:   r;   r<   r=   r   s          r    rm   $EarlyExitCandidateGenerator.__init__  sM     	+/%'- 	 	
 %)$:$:$O$O!6:3r#   r   c                   > [        U R                  U R                  R                  5      nUR                  R                  nU R
                  UR                  l        [        TU ]  U5      u  pEX2R                  l        XE4$ r   )getattrr9   base_model_prefixrS   num_hidden_layersr  r   r!   )r   r   
base_modeloriginal_num_hidden_layersrw   rx   r   s         r    r!   *EarlyExitCandidateGenerator.get_candidates  sn    T1143G3G3Y3YZ
%/%6%6%H%H".2.G.G
+*/'*@*K'.H+..r#   )r  r   )r,   r-   r.   r/   r0   r1   r2   r   r   rM   rm   r   r3   r!   r5   r  r  s   @r    r  r  g  s    : 1526;##; +; .	;
 ;  -; 0; ;,/(8(8 /U5CSCSU]^c^o^oUpCp=q / /r#   r  c           
      V   / n[        U[        5      (       a  UR                  U5        U$ U R                  R                  (       ao  [        [        U5      5       HJ  nUR                  X   S   SS2SS2SU2SS24   X   S   SS2SS2SU2SS24   X   S   X   S   45        ML     [        U5      nU$ SU R                  R                  R                  5       ;   dB  U R                  R                  b  SU R                  R                  S   R                  5       ;   a}  U R                  R                  (       a/  [        [        U5      5       H  nX   SS2SU2SS24   X'   M     U$ [        [        U5      5       H  nX   SS2SS2SU2SS24   X'   M      U$ Ub  [        [        U5      5       Hg  nX   / / 4:w  a=  UR                  X   S   SS2SS2SU2SS24   X   S   SS2SS2SU2SS24   45        MJ  UR                  X   S   X   S   45        Mi     [        U5      nU$ )z9Crops the past key values up to a certain maximum length.r   Nr   r   r~   
gptbigcode)rL   r   croprS   rT   r   r   r   r   r   r,   lowerarchitecturesmulti_query)modelr@   r   new_pastr   s        r    r   r     s9   H/5))Z(D C 
	(	(_-.COO#(+Aq+:+q,@A#(+Aq+:+q,@A#(+#(+	 /  /. + 
11779	9"".<5<<C]C]^_C`CfCfCh3h<<##S12'6';A{
{A<M'N$ 3"  S12'6';Aq+:+q<P'Q$ 3  
	$_-.C#Bx/',Q/1kzk10DE',Q/1kzk10DE !5a!8/:Nq:Q RS /  /r#   r;   
new_lengthrT   r   c                 |   U(       a  SOSnX0;  a  U $ X   nXR                   S   -
  nUS:  a  USS2SU24   X'   O<US:  a6  [        R                  " XDR                  UR                   S   U45      /SS9X'   SU ;   a_  U S   nUS:  a  USS2SU24   U S'   U $ US:  a<  USS2SS2SS2SS24   R	                  SUSS5      n[        R                  " Xg/SS9U S'   U $ S	U ;   aY  U S	   nUS:  a  USS2SU24   U S	'   U $ US:  a8  USS2SS2SS24   R	                  SUS5      n[        R                  " Xg/SS9U S	'   U $ )
zNExpands or crops the model's mask for decoding purposes, to the defined lengthrC   rF   r   r   Nr   r   cross_attention_maskimage_attention_mask)r\   r1   r   new_onesrepeat)r;   r  rT   mask_keyr  mask_length_diff
cross_masknew_masks           r    r   r     s    ,>'CSH#!D!JJqM1!!%a):*:):&:!;	A	!&D--AP`@a2b+cik!l -!"89
a3=aARBRAR>R3SL/0  !!!RS!Q,/66q:JAqQH3899j=SYZ3[L/0  
 <	/!"89
a3=aARBRAR>R3SL/0
 	 !!!RS!),33A7GKH3899j=SYZ3[L/0r#   c                    SU ;  d  U S   c  U $ U S   nUSS2S4   R                  S5      nXR                  S   -
  nUS:  a  USS2SU24   nU $ US:  a/  UR                  SU5      n[        R                  " U S   U/SS9U S'   U $ )zXExpands or crops the model's token_type_ids for decoding purposes, to the defined lengthtoken_type_idsNr   r   r   r   )r/  r\   r  r1   r   )r;   r  r  final_token_typetype_length_difftoken_type_copiess         r    r   r     s    |+|<L/M/U!"23N%ae,66r:!$8$8$;;!'+<,<+<(<=  
A	,33A7GH).LAQ4RTe3fln)o%&r#   )3rO   r  typingr   r   r   r   r   numpyr   r1   torch.nnr2  pytorch_utilsr	   utilsr
   sklearn.metricsr   cache_utilsr   r   logits_processr   r   r   modeling_utilsr   tokenization_utils_baser   configuration_utilsr   utils.deprecationr   r   r7   r   Moduler  r)  r5  r  r  r  r  r   r  r4   r   r   r   r+   r#   r    <module>r     sJ      < <    . ( )  - h h 0A5 /$
 $
N{/!3 {/|W4N WtRYY .5 5Dc cL6( 6(rq>,Y q>hm%7 m`5/"< 5/p&R!$sCx. !c !_c !hlmprumuhv !H$sCx. c dSVX[S[n r#   