
    fTh/}                        S SK Jr  S SKJrJrJrJr  S SKrS SKJ	r	  S SK
rSSKJr  SSKJrJrJr  SSKJr  SSKJr  SS	KJrJr  SS
KJr  SSKJr  SSKJrJr  SSKJ r   SSK!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*  \" 5       (       a  S SK+J,r,  SSK-J.r.  \R^                  " \05      r1 " S S\5      r2 " S S\(5      r3 " S S\&5      r4   S/S\	Rj                  S\Rl                  S\Rl                  S\Rl                  S\\Rl                     S\7S\\7   S\\7   S \\Rl                  \Rl                  4   4S! jjr8 " S" S#\"5      r9 " S$ S%\	Rj                  5      r: " S& S'\'5      r; " S( S)\#5      r< " S* S+\$5      r= " S, S-\%5      r>/ S.Qr?g)0    )partial)CallableOptionalTupleUnionN   )ACT2FN)CacheHybridCacheStaticCache)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)is_torch_flex_attn_availablelogging)deprecate_kwarg   )	GemmaAttentionGemmaForCausalLMGemmaForSequenceClassificationGemmaForTokenClassificationGemmaMLP
GemmaModelGemmaRMSNormapply_rotary_pos_emb	repeat_kv)	BlockMask)make_flex_block_causal_maskc                      ^  \ rS rSrSrSrS/rSSSSSSSS.rS/S	/4S
S/S
/4S
/S
/4S.r                        SU 4S jjr	Sr
U =r$ )Gemma2Config6   a  
This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the Gemma2-7B.
e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.
Args:
    vocab_size (`int`, *optional*, defaults to 256000):
        Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`Gemma2Model`]
    hidden_size (`int`, *optional*, defaults to 2304):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 9216):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 26):
        Number of hidden layers in the Transformer decoder.
    num_attention_heads (`int`, *optional*, defaults to 8):
        Number of attention heads for each attention layer in the Transformer decoder.
    num_key_value_heads (`int`, *optional*, defaults to 4):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details checkout [this
        paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
        `num_attention_heads`.
    head_dim (`int`, *optional*, defaults to 256):
        The attention head dimension.
    hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
        The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
        if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
    max_position_embeddings (`int`, *optional*, defaults to 8192):
        The maximum sequence length that this model might ever be used with.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
        The epsilon used by the rms normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    pad_token_id (`int`, *optional*, defaults to 0):
        Padding token id.
    eos_token_id (`int`, *optional*, defaults to 1):
        End of stream token id.
    bos_token_id (`int`, *optional*, defaults to 2):
        Beginning of stream token id.
    tie_word_embeddings (`bool`, *optional*, defaults to `True`):
        Whether to tie weight embeddings
    rope_theta (`float`, *optional*, defaults to 10000.0):
        The base period of the RoPE embeddings.
    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
        Whether to use a bias in the query, key, value and output projection layers during self-attention.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    query_pre_attn_scalar (`float`, *optional*, defaults to 256): scaling factor used on the attention scores
    sliding_window (`int`, *optional*, defaults to 4096): in Gemma2, every other layer uses sliding window attention. This is the
        size of the sliding window.
    final_logit_softcapping (`float`, *optional*, defaults to 30.0): scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0): scaling factor when applying tanh softcapping on the attention scores.
    cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.

```python
>>> from transformers import Gemma2Model, Gemma2Config
>>> # Initializing a Gemma2 gemma2-7b style configuration
>>> configuration = Gemma2Config()
>>> # Initializing a model from the gemma2-7b style configuration
>>> model = Gemma2Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```gemma2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormc                 0  > [         TU ]  " SUUUUS.UD6  Xl        Xl        X l        X0l        X@l        XPl        Xpl        X`l	        Xl
        Xl        Xl        UU l        UU l        UU l        Xl        UU l        UU l        UU l        UU l        UU l        g )N)pad_token_idbos_token_ideos_token_idtie_word_embeddings )super__init__
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizenum_hidden_layersnum_attention_headshead_dimnum_key_value_headsinitializer_rangerms_norm_eps	use_cache
rope_thetaattention_biasattention_dropouthidden_activationquery_pre_attn_scalarsliding_windowfinal_logit_softcappingattn_logit_softcappingcache_implementation)selfr8   r:   r;   r<   r=   r?   r>   rF   r9   r@   rA   rB   r1   r3   r2   r4   rC   rD   rE   rG   rH   rI   rJ   rK   kwargs	__class__s                             a/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/gemma2/modular_gemma2.pyr7   Gemma2Config.__init__   s    8 	 	
%%% 3		

 	
 %'>$&!2!2#6  #6 !2("$,!2!2%:",'>$&<#$8!    )rD   rE   rJ   rK   rI   r>   rF   r:   r@   r;   r9   r=   r<   r?   rG   rA   rC   rH   rB   r8   )i  i 	  i $              gelu_pytorch_tanhi    g{Gz?gư>Tr      r   Tg     @F        rU   i   g      >@g      I@hybrid)__name__
__module____qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr7   __static_attributes____classcell__rN   s   @rO   r#   r#   6   s    FP J#4"5%.%.%.%."+ )"+ &(9:#%568IJ!"_$56 - $ ! $#%369 69rQ   r#   c                       \ rS rSrSrg)Gemma2RMSNorm   r5   N)rZ   r[   r\   r]   rc   r5   rQ   rO   rg   rg      s    rQ   rg   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )	Gemma2MLP   c                 R   > [         TU ]  5         [        UR                     U l        g N)r6   r7   r	   rF   act_fnrL   configrN   s     rO   r7   Gemma2MLP.__init__   s     V556rQ   )rn   rZ   r[   r\   r]   r7   rc   rd   re   s   @rO   rj   rj      s    7 7rQ   rj   modulequerykeyvaluer,   dropoutscalingsoftcapreturnc                    Uc  U R                   S-  n[        X R                  5      n	[        X0R                  5      n
[        R                  " XR                  SS5      5      U-  nUb  X-  n[        R                  " U5      nX-  nUb"  US S 2S S 2S S 2S U	R                  S   24   nX-   n[        R                  R                  US[        R                  S9R                  UR                  5      n[        R                  R                  XU R                  S9n[        R                  " X5      nUR                  SS5      R!                  5       nX4$ )	N      r   r   )dimdtype)ptrainingrW   )r>   r   num_key_value_groupstorchmatmul	transposetanhshapenn
functionalsoftmaxfloat32tor   rw   r   
contiguous)rs   rt   ru   rv   r,   rw   rx   ry   rM   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                 rO   eager_attention_forwardr      s/    //4'3 ; ;<JU$?$?@L<<';';Aq'ABWLL#-zz,/#-!$Q1.D
0@0@0D.D%DE#1 ==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$rQ   c                   B  ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\	\R                  \R                  4   S\
\R                     S\
\   S	\
\R                     S
\\   S\	\R                  \
\R                     \
\	\R                        4   4S jjrSrU =r$ )Gemma2Attention   rp   	layer_idxc                   > [         TU ]  X5        U R                  R                  U l        U R                  R                  U l        SU l        UR                  S-  U l        [        US-  5      (       d  UR                  U l	        g S U l	        g )NTr|   r   )
r6   r7   rp   rJ   rE   	is_causalrG   rx   boolrH   rL   rp   r   rN   s      rO   r7   Gemma2Attention.__init__   sq    +&*kk&H&H#!%!>!>33T9;?	A;N;Nf33TXrQ   r+   position_embeddingsr,   past_key_valuecache_positionrM   rz   c                    UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      R                  U5      R	                  SS5      nUu  p[        XX5      u  pUb}  UUUU R                  S.nUR                  XU R                  U5      u  pUbJ  U R                  R                  S:X  a0  UR                   S   nU
S S 2S S 2S U2S S 24   US S 2S S 2S U2S S 24   p[        nU R                  R                  S:w  ad  U R                  R                  S:X  a-  UR                  SS	5      (       a  [        R!                  S
5        O["        U R                  R                     nU" U U	U
UU4U R$                  (       a  U R&                  OSU R(                  U R                  U R*                  S.UD6u  nnUR,                  " / UQSP76 R/                  5       nU R1                  U5      nUU4$ )Nr~   rW   r   )sincosr   rH   flash_attention_2eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rX   )rw   rx   rH   ry   )r   r>   q_projviewr   k_projv_projr   rH   updater   rp   _attn_implementationr   getloggerwarning_oncer   r   rE   rx   rJ   reshaper   o_proj)rL   r+   r   r,   r   r   rM   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsseq_lenattention_interfacer   r   s                      rO   forwardGemma2Attention.forward   sS    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&#7RU#[ % "0"&"5"5	L (6'<'<ZW[WeWegs't$J )dkk.N.NRe.e(..r2+5aHWHa6G+H,WXZ[]e^e]eghWhJiL(?;;++w6{{//69fjjI\^c>d>d##L
 '>dkk>^>^&_#$7%
 /3mmD**LL..//%
 %
!\ "));;;;FFHkk+.L((rQ   )rE   rJ   r   rx   rH   )NN)rZ   r[   r\   r]   r#   intr7   r   Tensorr   r   r
   
LongTensorr   r   r   rc   rd   re   s   @rO   r   r      s    Y| Y Y +/59;)||;) #5<<#=>;) !.	;)
 !;) !!1!12;) -.;) 
u||Xell3XeELL>Q5RR	S;) ;)rQ   r   c                     ^  \ rS rSrS\S\4U 4S jjr\" SSS9      SS\R                  S	\
\R                  \R                  4   S
\\R                     S\\R                     S\\   S\\   S\\   S\\R                     S\
\R                  \\
\R                  \R                  4      4   4S jj5       rSrU =r$ )Gemma2DecoderLayeri=  rp   r   c                   > [         TU ]  5         UR                  U l        Xl        [	        US-  5      (       + U l        [        XS9U l        [        U5      U l	        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        UR                   U l        g )Nr   )rp   r   )eps)r6   r7   r:   rp   r   
is_slidingr   	self_attnrj   mlprg   rA   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormrH   r   s      rO   r7   Gemma2DecoderLayer.__init__>  s    !--"9q=11(LV$,V-?-?VEXEXY(5f6H6HfNaNa(b%)6v7I7IvObOb)c&*78J8JPVPcPc*d'$33rQ   last_cache_positionz4.53.0)versionr+   r   r,   position_idsr   r   rB   r   rz   c	                    U R                   (       Ga8  UGb4  [        UR                  S   U R                  5      n
U R                  R
                  S:X  a  US S 2U
* S 24   nO[        R                  " UR                  5      R                  n[        R                  " [        R                  " U[        R                  S9U R                  * S9n[        R                  " XU5      nUS   U
-
  S-   n[        R                  " USS9n[        R                  " [        XR                  S   5      UR                   S9nX-  nUS S 2S S 2S S 2U4   nUnU R#                  U5      nU R$                  " S
UUUUUUUUS	.U	D6u  nnU R'                  U5      nX-   nUnU R)                  U5      nU R+                  U5      nU R-                  U5      nX-   nU4nU(       a  UU4-  nU$ )Nr   r   r   )diagonalr~   rW   )mindevice)r+   r   r,   r   r   r   rB   r   r5   )r   maxr   rH   rp   r   r   finfor   r   tril	ones_liker   whereclamparanger   r   r   r   r   r   r   )rL   r+   r   r,   r   r   r   rB   r   rM   effective_seq_len	min_dtypesliding_window_maskoffsetmask_indexesresidualself_attn_weightsoutputss                     rO   r   Gemma2DecoderLayer.forwardL  s    ???~9 #N$8$8$;T=P=P Q {{//3FF!/4E3E3F0F!G "KK(<(<=AA	&+jjOON%**EQUQdQdPd'# "'-@^!\'+.??!CV3  %||)+?+?+CD^MbMb  &!/1a0E!F ,,]; ,0>> 
,
' 3)%)/)
,
 
,
(( 55mD 0 66}E/77F 0 ")++GrQ   )
rp   r:   r   r   r   r   r   r   r   rH   )NNNFFN)rZ   r[   r\   r]   r#   r   r7   r   r   r   r   r   r   r
   r   FloatTensorr   rc   rd   re   s   @rO   r   r   =  s   4| 4 4 *H=
 2637*.,1$)59E||E #5<<#=>E !.	E
 u//0E !E $D>E D>E !!1!12E 
u  (51B1BEDUDU1U+V"WW	XE >ErQ   r   c                     ^  \ rS rSrS\4U 4S jjr         SS\\R                     S\\R                     S\\R                     S\\
   S\\R                     S	\\   S
\\   S\\   S\\R                     S\\   S\4S jjr\R"                  " 5        SS\\R                  S4   S\R                  S\R                  S\
S
\4
S jj5       rSrU =r$ )Gemma2Modeli  rp   c           	         > [         TU ]  U5        [        R                  " [	        UR
                  5       Vs/ s H  n[        X5      PM     sn5      U l        g s  snf rm   )r6   r7   r   
ModuleListranger<   r   r.   r   s      rO   r7   Gemma2Model.__init__  sH     mmDI&JbJbDcdDcy2Dcd
ds   Ar)   r,   r   r&   r*   rB   r   output_hidden_statesr   flash_attn_kwargsrz   c
                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUS L US L-  (       a  [	        S5      eU R
                  (       a/  U R                  (       a  U(       a  [        R                  S5        SnUc  U R                  U5      nU(       aN  UcK  U R                  (       d:  UR                  u  pn[        U R                   UUUR                  U R                  S9nU	cD  Ub  UR                  5       OSn[        R                   " XUR                  S   -   UR                  S9n	Uc  U	R#                  S5      nU R%                  X%XU5      nUnU R'                  UU5      n[        R(                  " U R                   R*                  S-  UR                  S	9nUU-  nU(       a  S
OS nU(       a  S
OS nU R,                  S U R                   R.                    H  nU(       a  UU4-  nU R
                  (       a?  U R                  (       a.  U R1                  [3        UR4                  40 U
D6UUUUUUUU	5	      nOU" U4UUUUUUU	S.U
D6nUS   nU(       d  M  UUS   4-  nM     U R7                  U5      nU(       a  UU4-  n[9        UUUUS9$ )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)max_batch_sizemax_cache_lenr   r   r   rW   r   g      ?r   r5   )r   r,   r   r   r   rB   r   )last_hidden_stater&   r+   
attentions)rp   r   r   rB   
ValueErrorgradient_checkpointingr   r   r   r-   r   r   r   r   get_seq_lengthr   r   	unsqueeze_update_causal_mask
rotary_embtensorr:   r.   r<   _gradient_checkpointing_funcr   __call__r/   r   )rL   r)   r,   r   r&   r*   rB   r   r   r   r   
batch_sizer   _past_seen_tokensr   r+   r   
normalizerall_hidden_statesall_self_attnsdecoder_layerlayer_outputss                          rO   r   Gemma2Model.forward  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==Yj I  --i8M0%2%8%8"J))%#)){{O !CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L..>L]

 & #oom\J
 \\$++"9"93">mFYFYZ
%
2 #7BD0d![[)H4;;+H+HIM#!m%55!**t}} $ A AM22H6GH!' #%"
! !.!
!(;#.!-#2&7'#1
! (
! *!,M  =#3"55A JD 		-0-!11&+++%	
 	
rQ   r    input_tensorc           
         U R                   R                  S:X  a  U$ U R                   R                  S:X  a,  [        U[        R                  5      (       a  [        U5      nU$ UR                  UR                  pvUR                  S   n[        U[        [        45      (       a  UR                  5       n	O!Ub  UR                  S   OUR                  S   n	U R                  UUU	UUUUR                  S   S9n
U
$ )Nr   flex_attentionrW   r~   r   sequence_lengthtarget_lengthr   r   r   r   )rp   r   
isinstancer   r   r!   r   r   r   r   r   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_position)rL   r,   r  r   r&   r   r   r   r  r  r   s              rO   r   Gemma2Model._update_causal_mask  s     ;;++/BB!!;;++/??.%,,77!<^!L!!$**L,?,?v&,,Q/o['ABB+??AM8F8RN004XdXjXjklXmM PP+')#))!, Q 
 rQ   )r.   )	NNNNNNNNN)F)rZ   r[   r\   r]   r#   r7   r   r   r   r   r   r   r   r   r   r   r   no_gradr   r   rc   rd   re   s   @rO   r   r     s[   
| 
 1515371559$(,0/359s
E,,-s
 !.s
 u//0	s

 "+.s
   1 12s
 D>s
 $D>s
 'tns
 !!1!12s
 $$89s
 
!s
j ]]_ #($ellK78$ ll$ 	$
 %$  $ $rQ   r   c                   l  ^  \ rS rSrU 4S jr           SS\\R                     S\\R                     S\\R                     S\\	   S\\R                     S\\R                     S	\\   S
\\   S\\   S\\R                     S\\\R                  4   S\4S jjr       SU 4S jjrSrU =r$ )Gemma2ForCausalLMi9  c                 d   > [         TU ]  U5        [        U5      U l        U R	                  5         g rm   r6   r7   r   model	post_initro   s     rO   r7   Gemma2ForCausalLM.__init__:  &      (
rQ   r)   r,   r   r&   r*   labelsrB   r   r   r   logits_to_keeprz   c                 F   U R                   (       aG  U R                  R                  S:w  a-  [        R	                  SU R                  R                   S35        Ub  UOU R                  R
                  nU	b  U	OU R                  R                  n	U R                  " SUUUUUUUU	U
S.	UD6nUR                  n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nU R                  R                  bH  UU R                  R                  -  n[        R                  " U5      nUU R                  R                  -  nSnUb  U R                   " UX`R"                  40 UD6n[%        UUUR&                  UR(                  UR*                  S9$ )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma2ForCausalLM

>>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r   zhIt is strongly recommended to train Gemma2 models with the `eager` attention implementation instead of `zp`. Use `eager` with `AutoModelForCausalLM.from_pretrained('<path-to-checkpoint>', attn_implementation='eager')`.N)	r)   r,   r   r&   r*   rB   r   r   r   )losslogitsr&   r+   r   r5   )r   rp   r   r   r   r   r   r  r   r	  r   slicelm_headrI   r   r   loss_functionr8   r   r&   r+   r   )rL   r)   r,   r   r&   r*   r  rB   r   r   r   r  loss_kwargsr   r+   slice_indicesr  r  s                     rO   r   Gemma2ForCausalLM.forward?  s   B ==T[[==H#{{??@  Aqr 2C1N-TXT_T_TqTq$8$D $++JjJj 	 ,0:: ,
)%+'/!5),
 ,
  118B>SV8W8W~ot4]kmA}a,?@A;;..:dkkAAAFZZ'FdkkAAAF%%ffooUUD%#33!//))
 	
rQ   c	                   > [         TU ]  " U4UUUUUUUS.U	D6n
Uc  U
R                  SS 5      n[        U[        5      (       a  UR
                  S:X  a  U R                  R                  S:X  d  U
S   b"  U
S   R                  u  pnU
S   R                  nO U
S   R                  u  pU
S   R                  nU R                  R                  UUUR                  5       U R                  R                  R                  UUUS9nX:S'   U
$ )	N)r&   r,   r*   r   r   rB   r  r  r   r   r*   r)   r  r,   )r6   prepare_inputs_for_generationpopr	  r   ndimrp   r   r   r   r  r  r
  r  weightr   )rL   r)   r&   r,   r*   r   r   rB   r  rM   model_inputsr   r   r  r   rN   s                  rO   r"  /Gemma2ForCausalLM.prepare_inputs_for_generation  s2    w<

+)')%)

 

 !  !148A 44##q(KK448KKO,81=o1N1T1T.
Q%o6==.:;.G.M.M+
%k299!ZZ]] /-AACll))//-% ^ N .<)*rQ   r  )NNNNNNNNNNr   )NNNNNTN)rZ   r[   r\   r]   r7   r   r   r   r   r   r   r   r   r   r   r   r"  rc   rd   re   s   @rO   r  r  9  s4    1515371559-1$(,0/35934K
E,,-K
 !.K
 u//0	K

 "+.K
   1 12K
 ))*K
 D>K
 $D>K
 'tnK
 !!1!12K
 c5<</0K
 
 K
` 4 4rQ   r  c                   (   ^  \ rS rSrU 4S jrSrU =r$ )Gemma2ForSequenceClassificationi  c                 d   > [         TU ]  U5        [        U5      U l        U R	                  5         g rm   r  ro   s     rO   r7   (Gemma2ForSequenceClassification.__init__  r  rQ   r(  rr   re   s   @rO   r*  r*         rQ   r*  c                   (   ^  \ rS rSrU 4S jrSrU =r$ )Gemma2ForTokenClassificationi  c                 d   > [         TU ]  U5        [        U5      U l        U R	                  5         g rm   r  ro   s     rO   r7   %Gemma2ForTokenClassification.__init__  r  rQ   r(  rr   re   s   @rO   r/  r/    r-  rQ   r/  )r#   r  r   Gemma2PreTrainedModelr*  r/  )rX   NN)@	functoolsr   typingr   r   r   r   r   torch.nnr   torch.utils.checkpointactivationsr	   cache_utilsr
   r   r   configuration_utilsr   modeling_flash_attention_utilsr   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.deprecationr   gemma.modeling_gemmar   r   r   r   r   r   r   r   r   !torch.nn.attention.flex_attentionr    integrations.flex_attentionr!   
get_loggerrZ   r   r#   rg   rj   Moduler   floatr   r   r   r   r  r*  r/  __all__r5   rQ   rO   <module>rG     s     3 3    ! : : 3 B O 5 & : 0
 
 
  !!;J 
		H	%P9# P9f	L 	7 7 ## %II %<< % 
 % <<	 %
 U\\* %  % e_ % e_ % 5<<%& %FD)n D)NU Upa* aHG( GT&D #> rQ   