
    fTh$                     &   S SK JrJrJrJr  S SKrS SKrS SKJr  SSKJ	r	  SSK
JrJr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJrJrJrJrJr  SSKJrJr  SSKJrJ r   SSK!J"r"  SSK#J$r$  SSK%J&r&J'r'J(r(J)r)J*r*  SSK+J,r,  \)" 5       (       a  S SK-J.r.  SSK/J0r0  SSK1J2r2  \*Rf                  " \45      r5\2" S5       " S S\Rl                  5      5       r7\$Rp                  " \75         " S S\Rl                  5      r9S r:S@S jr; " S S\Rl                  5      r<S\Rz                  S \>S!\Rz                  4S" jr? SAS#\Rl                  S$\Rz                  S%\Rz                  S&\Rz                  S'\\Rz                     S(\@S)\@4S* jjrA " S+ S,\Rl                  5      rB " S- S.\5      rC\' " S/ S0\ 5      5       rD\' " S1 S2\D5      5       rE " S3 S4\\&5      rF\' " S5 S6\D\5      5       rG\'" S7S89 " S9 S:\D5      5       rH\' " S; S<\D5      5       rI\' " S= S>\D5      5       rJ/ S?QrKg)B    )CallableOptionalTupleUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ALL_LAYERNORM_LAYERS)
LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging   )LlamaConfig)	BlockMask)make_flex_block_causal_mask)use_kernel_forward_from_hubRMSNormc                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )LlamaRMSNorm:   c                    > [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z+
LlamaRMSNorm is equivalent to T5LayerNorm
N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__s      `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/llama/modeling_llama.pyr+   LlamaRMSNorm.__init__<   s/     	ll5::k#:; #    c                    UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )N   T)keepdim)	dtypetor-   float32powmeanrsqrtr0   r/   )r1   hidden_statesinput_dtypevariances       r5   forwardLlamaRMSNorm.forwardD   sw    #))%((7 $$Q',,R,>%H?T?T4T(UU{{]--k:::r7   c                 ^    [        U R                  R                  5       SU R                   3$ )Nz, eps=)tupler/   shaper0   r1   s    r5   
extra_reprLlamaRMSNorm.extra_reprK   s*    ))*+6$2G2G1HIIr7   )r0   r/   )gư>)	__name__
__module____qualname____firstlineno__r+   rE   rK   __static_attributes____classcell__r4   s   @r5   r'   r'   :   s    $;J Jr7   r'   c                   l   ^  \ rS rSrSS\4U 4S jjjr\R                  " 5       \S 5       5       r	Sr
U =r$ )LlamaRotaryEmbeddingR   configc                   > [         TU ]  5         [        US5      (       aH  UR                  b;  UR                  R	                  SUR                  R	                  S5      5      U l        OSU l        UR                  U l        UR                  U l        Xl	        [        U R
                     U l        U R                  U R                  U5      u  o0l        U R                  SUSS9  U R                  U l        g )Nrope_scaling	rope_typetypedefaultinv_freqF)
persistent)r*   r+   hasattrrY   getrZ   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrW   r   rope_init_fnattention_scalingregister_bufferr]   original_inv_freq)r1   rW   devicer]   r4   s       r5   r+   LlamaRotaryEmbedding.__init__S   s    6>**v/B/B/N#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q((ZeD!%r7   c                 b   U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      R	                  UR
                  5      nUS S 2S S S 24   R                  5       n[        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        R                  " USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " Xf4SS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR	                  UR                   S
9W	R	                  UR                   S
94$ ! , (       d  f       N@= f)Nr   r:   r    mpscpuF)device_typeenabledr9   dim)r<   )r]   floatexpandrI   r=   rh   
isinstancer[   strr-   autocast	transposecatcosre   sinr<   )
r1   xposition_idsinv_freq_expandedposition_ids_expandedrm   freqsembrx   ry   s
             r5   rE   LlamaRotaryEmbedding.forwardd   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk^^UC&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d444C'')d444C	 D vvAGGv$cff177f&;;; DCs   $BF  
F.)re   rW   rb   rg   rc   rd   rZ   N)rM   rN   rO   rP   r!   r+   r-   no_gradr   rE   rQ   rR   rS   s   @r5   rU   rU   R   s6    /{ / /" ]]_<  <r7   rU   c                     U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nr:   r9   ro   )rI   r-   rw   )rz   x1x2s      r5   rotate_halfr   t   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r7   c                     UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXg4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)	unsqueezer   )qkrx   ry   r{   unsqueeze_dimq_embedk_embeds           r5   apply_rotary_pos_embr   {   sS    ( --
&C
--
&Cw;q>C/0Gw;q>C/0Gr7   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )LlamaMLP   c                   > [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  UR                  S9U l        [
        R                  " U R                  U R                  UR                  S9U l	        [
        R                  " U R                  U R                  UR                  S9U l
        [        UR                     U l        g )Nbias)r*   r+   rW   r2   intermediate_sizer   Linearmlp_bias	gate_projup_proj	down_projr	   
hidden_actact_fnr1   rW   r4   s     r5   r+   LlamaMLP.__init__   s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r7   c                     U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r   )r   r   r   r   )r1   rz   r   s      r5   rE   LlamaMLP.forward   s6    NN4;;t~~a/@#ADLLQRO#ST	r7   )r   rW   r   r   r2   r   r   )rM   rN   rO   rP   r+   rE   rQ   rR   rS   s   @r5   r   r      s    0 r7   r   rB   n_repreturnc                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r    N)rI   rr   reshape)rB   r   batchnum_key_value_headsslenhead_dims         r5   	repeat_kvr      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr7   modulequerykeyvalueattention_maskscalingdropoutc                 @   [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub"  US S 2S S 2S S 2S UR
                  S   24   nX-   n
[        R                  R                  U
S[        R                  S9R                  UR                  5      n
[        R                  R                  XU R                  S9n
[        R                  " X5      nUR	                  SS5      R                  5       nX4$ )Nr9   r   r:   )rp   r<   )ptrainingr    )r   num_key_value_groupsr-   matmulrv   rI   r   
functionalsoftmaxr>   r=   r<   r   r   
contiguous)r   r   r   r   r   r   r   kwargs
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r5   eager_attention_forwardr      s     3 ; ;<JU$?$?@L<<';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#1==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$r7   c                   F  ^  \ rS rSrSrS\S\4U 4S jjr  SS\R                  S\
\R                  \R                  4   S\\R                     S	\\   S
\\R                     S\\   S\
\R                  \\R                     \\
\R                        4   4S jjrSrU =r$ )LlamaAttention   z=Multi-headed attention from 'Attention Is All You Need' paperrW   	layer_idxc                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nr   g      Tr   )r*   r+   rW   r   getattrr2   num_attention_headsr   r   r   r   attention_dropout	is_causalr   r   attention_biasq_projk_projv_projo_projr1   rW   r   r4   s      r5   r+   LlamaAttention.__init__   sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r7   rB   position_embeddingsr   past_key_valuecache_positionr   r   c                    UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      R                  U5      R	                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  ad  U R                  R                  S:X  a-  UR                  SS5      (       a  [        R                  S	5        O[         U R                  R                     nU" U U	U
UU4U R"                  (       d  S
OU R$                  U R&                  S.UD6u  nnUR(                  " / UQSP76 R+                  5       nU R-                  U5      nUU4$ )Nr:   r    r9   )ry   rx   r   eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        )r   r   )rI   r   r   viewrv   r   r   r   updater   r   rW   _attn_implementationr`   loggerwarning_oncer   r   r   r   r   r   r   )r1   rB   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   rx   ry   cache_kwargsattention_interfacer   r   s                     r5   rE   LlamaAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&#7RU#[ %#&nUL'5'<'<ZW[WeWegs't$J(?;;++w6{{//69fjjI\^c>d>d##L
 '>dkk>^>^&_#$7	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r7   )r   rW   r   r   r   r   r   r   r   r   r   )NN)rM   rN   rO   rP   __doc__r!   intr+   r-   Tensorr   r   r
   
LongTensorr   r   rE   rQ   rR   rS   s   @r5   r   r      s    G
{ 
s 
8 +/590)||0) #5<<#=>0) !.	0)
 !0) !!1!120) -.0) 
u||Xell3XeELL>Q5RR	S0) 0)r7   r   c                     ^  \ rS rSrS\S\4U 4S jjr       SS\R                  S\	\R                     S\	\R                     S\	\   S	\	\   S
\	\   S\	\R                     S\	\\R                  \R                  4      S\\   S\\R                   \	\\R                   \R                   4      4   4S jjrSrU =r$ )LlamaDecoderLayeri  rW   r   c                   > [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N)rW   r   r3   )r*   r+   r2   r   	self_attnr   mlpr'   rms_norm_epsinput_layernormpost_attention_layernormr   s      r5   r+   LlamaDecoderLayer.__init__  sj    !--'vKF#+F,>,>FDWDWX(4V5G5GVM`M`(a%r7   rB   r   r{   r   r   	use_cacher   r   r   r   c	                     Un
U R                  U5      nU R                  " SUUUUUUUUS.U	D6u  pX-   nUn
U R                  U5      nU R                  U5      nX-   nU4nU(       a  X4-  nU$ )N)rB   r   r{   r   r   r   r   r    )r   r   r   r   )r1   rB   r   r{   r   r   r   r   r   r   residualself_attn_weightsoutputss                r5   rE   LlamaDecoderLayer.forward$  s     !,,]; ,0>> 
,
')%)/) 3
,
 
,
( !0 !55mD/ 0 "++Gr7   )r2   r   r   r   r   )NNNFFNN)rM   rN   rO   rP   r!   r   r+   r-   r   r   r   r
   boolr   r   r   FloatTensorrE   rQ   rR   rS   s   @r5   r   r     s   b{ bs b 2637*.,1$)59KO'||' !.' u//0	'
 !' $D>' D>' !!1!12' &eELL%,,,F&GH' -.' 
u  (51B1BEDUDU1U+V"WW	X' 'r7   r   c                   N    \ rS rSr\rSrSrS/rS/r	Sr
SrSrSrSrSrSrS rSrg)	LlamaPreTrainedModeliN  modelTr   past_key_valuesc                    U R                   R                  n[        U[        R                  5      (       aW  UR
                  R                  R                  SUS9  UR                  b%  UR                  R                  R                  5         g g [        U[        R                  5      (       ad  UR
                  R                  R                  SUS9  UR                  b2  UR
                  R                  UR                     R                  5         g g [        U[        5      (       a&  UR
                  R                  R                  S5        g g )Nr   )r@   stdg      ?)rW   initializer_rangers   r   r   r/   datanormal_r   zero_	Embeddingpadding_idxr'   fill_)r1   r   r   s      r5   _init_weights"LlamaPreTrainedModel._init_weights]  s    kk++fbii((MM&&CS&9{{&  &&( '--MM&&CS&9!!-""6#5#56<<> .--MM$$S) .r7   r   N)rM   rN   rO   rP   r!   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_flex_attn_supports_cache_class_supports_quantized_cache_supports_static_cache_supports_attention_backendr  rQ   r   r7   r5   r   r   N  sS    L&*#,-#4"5!N  $!"&*r7   r   c                     ^  \ rS rSrS\4U 4S jjrS rS r\\	         SS\
\R                     S\
\R                     S\
\R                     S	\
\   S
\
\R                     S\
\   S\
\   S\
\   S\
\R                     S\\   S\4S jj5       5       r SS\\R                  S4   S\R                  S\R                  S	\S\4
S jjr\S\R                  S\S\S\R2                  S\R                  S\4S j5       rSrU =r$ )
LlamaModelik  rW   c           	        > [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr   )rW   F)r*   r+   pad_token_idr  
vocab_sizer   r  r2   embed_tokens
ModuleListrangenum_hidden_layersr   layersr'   r   normrU   
rotary_embgradient_checkpointing	post_initr   s      r5   r+   LlamaModel.__init__m  s     !.. ++LL):):F<N<NPTP`P`ammCHIaIaCbcCbiv1Cbc
 !!3!39L9LM	.f=&+# 	 ds   C?c                     U R                   $ r   r  rJ   s    r5   get_input_embeddingsLlamaModel.get_input_embeddings}  s       r7   c                     Xl         g r   r&  r1   r   s     r5   set_input_embeddingsLlamaModel.set_input_embeddings  s    !r7   	input_idsr   r{   r   inputs_embedsr   r   output_hidden_statesr   flash_attn_kwargsr   c
                 J   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUS L US L-  (       a  [	        S5      eU R
                  (       a/  U R                  (       a  U(       a  [        R                  S5        Sn[        U[        S 5      [        45      (       d  [	        S5      eUc  U R                  U5      nU(       a  Uc
  [        5       nU	cD  Ub  UR                  5       OSn[        R                   " XUR"                  S   -   UR$                  S9n	Uc  U	R'                  S5      nU R)                  X%XU5      nUnU R+                  X5      nU(       a  SOS nU(       a  SOS nU R,                  S U R                   R.                    H7  nU(       a  X4-  nU" U4UUUUUU	US	.U
D6nUS   nU(       d  M.  UUS   4-  nM9     U R1                  U5      nU(       a  X4-  n[3        UU(       a  UOS UUS
9$ )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBThe `past_key_values` should be either a `Cache` object or `None`.r   r    rh   r   )r   r{   r   r   r   r   r   )last_hidden_stater   rB   
attentions)rW   r   r/  r   
ValueErrorr"  r   r   r   rs   r[   r
   r  r   get_seq_lengthr-   arangerI   rh   r   _update_causal_maskr!  r  r  r   r   )r1   r-  r   r{   r   r.  r   r   r/  r   r0  past_seen_tokensr   rB   r   all_hidden_statesall_self_attnsdecoder_layerlayer_outputss                      r5   rE   LlamaModel.forward  sI    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==Yj I /DJ+>??abb  --i8M0*nO!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L..>L]
 & #oomJ #7BD0d![[)H4;;+H+HIM#!%55!)
*)."3#-$7
 $
M *!,M  =#3"55' J* 		-0  !11&+/8Od+%	
 	
r7   r"   input_tensorc           	         U R                   R                  S:X  a  Ub  US:H  R                  5       (       a  U$ g U R                   R                  S:X  a,  [        U[        R
                  5      (       a  [        U5      nU$ Ub  UR                  5       OSnUb  UR                  OSnU R                   R                  S:X  a5  U(       d.  U(       d'  [        R                  " UUUU R                  S9(       a  g UR                  nUR                  S   n	U(       a  UR                  5       n
O5[        U[        R
                  5      (       a  UR                  S	   OXi-   S-   n
U R                  UU	U
UUUR                  S   S
9nU R                   R                  S:X  aZ  UbW  UR                   R"                  S;   a=  U(       d6  [        R$                  " U5      R&                  n[        R(                  " X5      nU$ )Nflash_attention_2r   flex_attentionr   Fr   )r.  past_key_values_lengthis_trainingr    r:   )sequence_lengthtarget_lengthr<   r   
batch_size)cudaxpunpu)rW   r   anyrs   r-   r   r#   r6  is_compileabler   _ignore_causal_mask_sdpar   r<   rI   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionrh   r[   finfomin_unmask_unattended)r1   r   r?  r   r   r   r9  using_compilable_cacher<   rE  rF  r   	min_dtypes                r5   r8  LlamaModel._update_causal_mask  s    ;;++/BB)~/D.I.I.K.K%%;;++/??.%,,77!<^!L!!
 @O?Z?99;`aCRC^!?!?di ;;++v5>T]n%>>*'7 MM	 ""&,,Q/!+??AM nell;; $$R(%7!;  PP+')#))!, Q 
 KK,,6*%%**.DD%
 E*..I0CCK[Kr7   rE  rF  r<   rG  c                    U b  U R                  5       S:X  a  U nU$ [        R                  " U5      R                  n[        R                  " X4XUR
                  S9nUS:w  a  [        R                  " USS9nU[        R                  " X$R
                  S9UR                  SS5      :  -  nUSSSS2SS24   R                  USSS5      nU b  UR                  5       nU R                  S   n	USS2SS2SS2SU	24   U SS2SSSS24   R                  UR
                  5      -   n
U
S:H  n
USS2SS2SS2SU	24   R                  X5      USS2SS2SS2SU	24'   U$ )	a  
Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

Args:
    attention_mask (`torch.Tensor`):
        A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
        `(batch_size, 1, query_length, key_value_length)`.
    sequence_length (`int`):
        The sequence length being processed.
    target_length (`int`):
        The target length: when generating with static cache, the mask should be as long as the static cache,
        to account for the 0 padding, the part of the cache that is not filled yet.
    dtype (`torch.dtype`):
        The dtype to use for the 4D attention mask.
    cache_position (`torch.Tensor`):
        Indices depicting the position of the input sequence tokens in the sequence.
    batch_size (`torch.Tensor`):
        Batch size.
N   )
fill_valuer<   rh   r    )diagonalr2  r:   r   )rp   r-   rP  rQ  fullrh   triur7  r   rr   clonerI   r=   masked_fill)r   rE  rF  r<   r   rG  r   r   rT  mask_lengthpadding_masks              r5   rO  @LlamaModel._prepare_4d_causal_attention_mask_with_cache_position'  s}   < %.*<*<*>!*C(K* ' E*..I** 0Y\j\q\qK !##jjqA5<<>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c 6Aq!\k\12 r7   )r  r"  r  r   r  r!  r  	NNNNNNNNN)F)rM   rN   rO   rP   r!   r+   r'  r+  r   r   r   r-   r   r   r
   r   r   r   r   r   rE   r   r8  staticmethodr   r<   rO  rQ   rR   rS   s   @r5   r  r  k  s   {  !"  151537+/59$(,0/359\
E,,-\
 !.\
 u//0	\

 "%\
   1 12\
 D>\
 $D>\
 'tn\
 !!1!12\
 $$89\
 
!\
  \
H #(BellK78B llB 	B
 B  BH 444 4 {{	4
 4 4 4r7   r  c                       \ rS rSrSrg)KwargsForCausalLMi_  r   N)rM   rN   rO   rP   rQ   r   r7   r5   rd  rd  _  s    3r7   rd  c                     ^  \ rS rSrS/rSS0rSS/S/40rU 4S jrS rS	 r	S
 r
S rS rS r\\           SS\\R$                     S\\R&                     S\\R$                     S\\   S\\R*                     S\\R$                     S\\   S\\   S\\   S\\R$                     S\\\R&                  4   S\\   S\4S jj5       5       rSrU =r$ )LlamaForCausalLMib  zlm_head.weightlm_headcolwise_reprB   logitsc                    > [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g NFr   )
r*   r+   r  r   r  r   r   r2   rg  r#  r   s     r5   r+   LlamaForCausalLM.__init__h  sU     '
 ++yy!3!3V5F5FUS 	r7   c                 .    U R                   R                  $ r   r   r  rJ   s    r5   r'  %LlamaForCausalLM.get_input_embeddingsq      zz&&&r7   c                 $    XR                   l        g r   rn  r*  s     r5   r+  %LlamaForCausalLM.set_input_embeddingst      "'

r7   c                     U R                   $ r   rg  rJ   s    r5   get_output_embeddings&LlamaForCausalLM.get_output_embeddingsw  s    ||r7   c                     Xl         g r   ru  )r1   new_embeddingss     r5   set_output_embeddings&LlamaForCausalLM.set_output_embeddingsz  s    %r7   c                     Xl         g r   r   )r1   decoders     r5   set_decoderLlamaForCausalLM.set_decoder}  s    
r7   c                     U R                   $ r   r}  rJ   s    r5   get_decoderLlamaForCausalLM.get_decoder  s    zzr7   r-  r   r{   r   r.  labelsr   r   r/  r   logits_to_keepr   r   c                    Ub  UOU R                   R                  nU	b  U	OU R                   R                  n	U R                  " SUUUUUUUU	U
S.	UD6nUR                  n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SUX`R                   R                  S.UD6n[        UUUR                  UR                  UR                  S9$ )a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, LlamaForCausalLM

>>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```N)	r-  r   r{   r   r.  r   r   r/  r   )ri  r  r  lossri  r   rB   r4  r   )rW   r   r/  r   r3  rs   r   slicerg  loss_functionr  r   r   rB   r4  )r1   r-  r   r{   r   r.  r  r   r   r/  r   r  r   r   rB   slice_indicesri  r  s                     r5   rE   LlamaForCausalLM.forward  s   N 2C1N-TXT_T_TqTq$8$D $++JjJj 	
 ,0:: ,
)%+'/!5),
 ,
  118B>SV8W8W~ot4]kmA}a,?@A%%pVF{{OeOepiopD%#33!//))
 	
r7   )rg  r   r  )NNNNNNNNNNr   )rM   rN   rO   rP   _tied_weights_keys_tp_plan_pp_planr+   r'  r+  rv  rz  r  r  r   r   r   r-   r   r   r
   r   r   r   r   r   rd  r   rE   rQ   rR   rS   s   @r5   rf  rf  b  s   *+=)H_-z:;H'(&  151537+/59-1$(,0/35934G
E,,-G
 !.G
 u//0	G

 "%G
   1 12G
 ))*G
 D>G
 $D>G
 'tnG
 !!1!12G
 c5<</0G
 *+G
 
 G
  G
r7   rf  a  
    The LLaMa Model transformer with a sequence classification head on top (linear layer).

    [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )custom_introc                   *  ^  \ rS rSrU 4S jrS rS r\\         SS\	\
R                     S\	\
R                     S\	\
R                     S\	\   S	\	\
R                     S
\	\
R                     S\	\   S\	\   S\	\   S\4S jj5       5       rSrU =r$ )LlamaForSequenceClassificationi  c                    > [         TU ]  U5        UR                  U l        [        U5      U l        [
        R                  " UR                  U R                  SS9U l        U R                  5         g rk  )
r*   r+   
num_labelsr  r   r   r   r2   scorer#  r   s     r5   r+   'LlamaForSequenceClassification.__init__  sS      ++'
YYv114??O
 	r7   c                 .    U R                   R                  $ r   rn  rJ   s    r5   r'  3LlamaForSequenceClassification.get_input_embeddings  rp  r7   c                 $    XR                   l        g r   rn  r*  s     r5   r+  3LlamaForSequenceClassification.set_input_embeddings  rs  r7   r-  r   r{   r   r.  r  r   r   r/  r   c
                    U R                  UUUUUUUU	S9n
U
R                  nU R                  U5      nUb  UR                  S   nOUR                  S   nU R                  R
                  c  US:w  a  [        S5      eU R                  R
                  c  SnOUb  XR                  R
                  :g  R                  UR                  [        R                  5      n[        R                  " UR                  S   UR                  [        R                  S9nUU-  R                  S5      nO.Sn[        R                  U R                  R                    S35        U[        R                  " XR                  S	9U4   nSnUb  U R#                  XUU R                  S
9n[%        UUU
R&                  U
R(                  U
R*                  S9$ )e  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r   r{   r   r.  r   r   r/  Nr   r    z=Cannot handle batch sizes > 1 if no padding token is defined.r:   )rh   r<   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r2  )ri  r  pooled_logitsrW   r  )r   r3  r  rI   rW   r  r5  r=   rh   r-   int32r7  argmaxr   r   r4   rM   r  r   r   rB   r4  )r1   r-  r   r{   r   r.  r  r   r   r/  transformer_outputsrB   ri  rG  last_non_pad_tokennon_pad_masktoken_indicesr  r  s                      r5   rE   &LlamaForSequenceClassification.forward  s   * 8<zz)%+'/!5 8B 	8
 ,==M* "+J&,,Q/J;;##+
a\]];;##+!#"%)A)AAEEfmmUZU`U`aL!LL)<V]]Z_ZeZefM"/,">!F!Fr!J!#>>**+ ,Z Z
 u||J}}MOaab%%VR_hlhshs%tD/ /??-;;*55
 	
r7   )r   r  r  ra  )rM   rN   rO   rP   r+   r'  r+  r   r   r   r-   r   r   r
   r   r   r   rE   rQ   rR   rS   s   @r5   r  r    s    '(  151537+/59-1$(,0/3A
E,,-A
 !.A
 u//0	A

 "%A
   1 12A
 ))*A
 D>A
 $D>A
 'tnA
 
*A
  A
r7   r  c                   B  ^  \ rS rSrSrU 4S jrS rS r\\	         SS\
\R                     S\
\R                     S\
\R                     S	\
\   S
\
\R                     S\
\R                     S\
\R                     S\
\   S\
\   S\4S jj5       5       rSrU =r$ )LlamaForQuestionAnsweringi3  transformerc                    > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  S5      U l        U R                  5         g )Nr9   )	r*   r+   r  r  r   r   r2   
qa_outputsr#  r   s     r5   r+   "LlamaForQuestionAnswering.__init__8  sA     %f-))F$6$6: 	r7   c                 .    U R                   R                  $ r   r  r  rJ   s    r5   r'  .LlamaForQuestionAnswering.get_input_embeddings@  s    ,,,r7   c                 $    XR                   l        g r   r  r*  s     r5   r+  .LlamaForQuestionAnswering.set_input_embeddingsC  s    (-%r7   r-  r   r{   r   r.  start_positionsend_positionsr   r/  r   c
           
         U R                  UUUUUUU	S9nUR                  nU R                  U5      nUR                  SSS9u  pUR	                  S5      R                  5       nUR	                  S5      R                  5       nS nUb  Ub  U R                  " XXg40 U
D6n[        UUUUR                  UR                  S9$ )N)r   r{   r   r.  r   r/  r    r:   ro   )r  start_logits
end_logitsrB   r4  )
r  r3  r  splitsqueezer   r  r   rB   r4  )r1   r-  r   r{   r   r.  r  r  r   r/  r   r   sequence_outputri  r  r  r  s                    r5   rE   !LlamaForQuestionAnswering.forwardF  s     ,0+;+;)%+'/!5 ,< ,
 "331#)<<r<#: #++B/::<''+668
&=+D%%libhiD+%!!//))
 	
r7   )r  r  ra  )rM   rN   rO   rP   r  r+   r'  r+  r   r   r   r-   r   r   r
   r   r   r   rE   rQ   rR   rS   s   @r5   r  r  3  s    %-.  151537+/596:48,0/3(
E,,-(
 !.(
 u//0	(

 "%(
   1 12(
 "%"2"23(
   0 01(
 $D>(
 'tn(
 
&(
  (
r7   r  c                   *  ^  \ rS rSrU 4S jrS rS r\\         SS\	\
R                     S\	\
R                     S\	\
R                     S\	\   S	\	\
R                     S
\	\
R                     S\	\   S\	\   S\	\   S\4S jj5       5       rSrU =r$ )LlamaForTokenClassificationis  c                   > [         TU ]  U5        UR                  U l        [        U5      U l        [        USS 5      b  UR                  nO[        USS 5      b  UR                  nOSn[        R                  " U5      U l
        [        R                  " UR                  UR                  5      U l        U R                  5         g )Nclassifier_dropouthidden_dropoutg?)r*   r+   r  r  r   r   r  r  r   Dropoutr   r   r2   r  r#  )r1   rW   r  r4   s      r5   r+   $LlamaForTokenClassification.__init__u  s      ++'
6/6B!'!:!:V-t4@!'!6!6!$zz"45YYv1163D3DE
 	r7   c                 .    U R                   R                  $ r   rn  rJ   s    r5   r'  0LlamaForTokenClassification.get_input_embeddings  rp  r7   c                 $    XR                   l        g r   rn  r*  s     r5   r+  0LlamaForTokenClassification.set_input_embeddings  rs  r7   r-  r   r{   r   r.  r  r   r   r/  r   c
                    U R                  UUUUUUUU	S9n
U
R                  nU R                  U5      nU R                  U5      nSnUb  U R	                  XU R
                  5      n[        UUU
R                  U
R                  S9$ )r  r  N)r  ri  rB   r4  )	r   r3  r   r  r  rW   r   rB   r4  )r1   r-  r   r{   r   r.  r  r   r   r/  r   r  ri  r  s                 r5   rE   #LlamaForTokenClassification.forward  s    * ,0::)%+'/!5 ,6 	,
 "33,,7O,%%fdkkBD$!//))	
 	
r7   )r   r   r  r  ra  )rM   rN   rO   rP   r+   r'  r+  r   r   r   r-   r   r   r
   r   r   r   rE   rQ   rR   rS   s   @r5   r  r  s  s     '(  151537+/59-1$(,0/3*
E,,-*
 !.*
 u//0	*

 "%*
   1 12*
 ))**
 D>*
 $D>*
 'tn*
 
*
  *
r7   r  )rf  r  r   r  r  r  )Nr    )r   )Ltypingr   r   r   r   r-   torch.utils.checkpointr   activationsr	   cache_utilsr
   r   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r   r   r   configuration_llamar!   !torch.nn.attention.flex_attentionr"   integrations.flex_attentionr#   integrationsr$   
get_loggerrM   r   Moduler'   appendrU   r   r   r   r   r   r   rq   r   r   r   r   r  rd  rf  r  r  r  __all__r   r7   r5   <module>r     sV  ( 4 3    ! . ) > B 9  L F & 1 h h ,  !!;J 7 
		H	% Y'J299 J (J(   L )<299 <D(6ryy  	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % %4J)RYY J)Z22 2j *? * *8 p% p pf ?,j > i
+_ i
 i
X S
%9 S
S
l <
 4 <
 <
~ C
"6 C
 C
Lr7   