o
    Zhu                     @   sn  d dl mZmZmZ d dlZd dlmZ d dlZddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% e&e'Z(G dd deZ)G dd de!Z*G dd deZ+G dd deej,Z-G dd deZ.G dd de Z/G dd de%Z0G dd deZ1g d Z2dS )!    )CallableOptionalTupleN   )CacheHybridCache)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputWithPast)rope_config_validation)ALL_ATTENTION_FUNCTIONS)Unpack)logging)deprecate_kwarg   )CohereAttentionCohereDecoderLayerCohereForCausalLMCohereLayerNormCoherePreTrainedModelCohereRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)Gemma2Modelc                       s   e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ																							d" fd d!	Z  Z	S )#Cohere2Configa  
    This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
    model according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.


    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`CohereModel`]
        hidden_size (`int`, *optional*, defaults to 8192):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 22528):
            Dimension of the MLP representations.
        logit_scale (`float`, *optional*, defaults to 0.0625):
            The scaling factor for the output logits.
        num_hidden_layers (`int`, *optional*, defaults to 40):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 64):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 5):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 255001):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
            accordingly.
            Expected contents:
                `rope_type` (`str`):
                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
                    'llama3'], with 'default' being the original RoPE implementation.
                `factor` (`float`, *optional*):
                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
                    original maximum pre-trained length.
                `original_max_position_embeddings` (`int`, *optional*):
                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
                    pretraining.
                `attention_factor` (`float`, *optional*):
                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
                    computation. If unspecified, it defaults to value recommended by the implementation, using the
                    `factor` field to infer the suggested value.
                `beta_fast` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
                    ramp function. If unspecified, it defaults to 32.
                `beta_slow` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
                    ramp function. If unspecified, it defaults to 1.
                `short_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `long_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `low_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
                `high_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        sliding_window (`int`, *optional*, defaults to 4096):
            Size of the sliding window attention context.
        sliding_window_pattern (`int`, *optional*, defaults to 4):
            Pattern for the sliding window attention.
        cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.

    ```python
    >>> from transformers import Cohere2Model, Cohere2Config

    >>> # Initializing a Cohere Nextmodel configuration
    >>> configuration = Cohere2Config()

    >>> # Initializing a model from the Cohere2 configuration
    >>> model = Cohere2Model(configuration) # doctest: +SKIP

    >>> # Accessing the model configuration
    >>> configuration = model.config # doctest: +SKIP
    ```
    Zcohere2past_key_valuesZcolwiseZrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm       X        ?(   @   Nsilu{Gz?h㈵>Tr            @F              hybridc                    s   || _ |	| _|| _|| _|| _|| _|| _|d u r|}|| _|| _|
| _	|| _
|| _|| _|| _|| _|| _|| _|| _|| | _|| _t|  t jd||||d| d S )N)pad_token_idbos_token_ideos_token_idtie_word_embeddings )
vocab_sizemax_position_embeddingshidden_sizelogit_scaleintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actinitializer_rangelayer_norm_eps	use_cache
rope_thetarope_scalingattention_biasattention_dropoutsliding_windowsliding_window_patternhead_dimcache_implementationr   super__init__)selfr8   r:   r<   r;   r=   r>   r?   r@   r9   rA   rB   rC   r3   r4   r5   r6   rD   rE   rF   rG   rH   rI   rK   kwargs	__class__r7   Z/var/www/auris/lib/python3.10/site-packages/transformers/models/cohere2/modular_cohere2.pyrM      s>   

zCohere2Config.__init__)r#   r$   r%   r&   r'   r(   Nr)   r$   r*   r+   Tr   r,   r-   Tr.   NFr/   r0   r1   r2   )
__name__
__module____qualname____doc__Z
model_typeZkeys_to_ignore_at_inferenceZbase_model_tp_planZbase_model_pp_planrM   __classcell__r7   r7   rP   rR   r   /   sP    q


r   c                   @      e Zd ZdS )Cohere2RotaryEmbeddingNrS   rT   rU   r7   r7   r7   rR   rY          rY   c                   @   rX   )Cohere2LayerNormNrZ   r7   r7   r7   rR   r\      r[   r\   c                   @   s   e Zd ZdZddedee fddZ		ddej	de
ej	ej	f d	eej	 d
ee deej dee de
ej	eej	 ee
ej	  f fddZdS )Cohere2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNconfig	layer_idxc                 C   s   t j  || _|| _t|d|j|j | _|j|j	 | _
| jd | _|j| _d| _t j|j|j| j |jd| _t j|j|j	| j |jd| _t j|j|j	| j |jd| _t j|j| j |j|jd| _| jd | jj dkrw|j| _d S d | _d S )NrJ   g      T)Zbias   r   )nnModulerM   r^   r_   getattrr:   r>   rJ   r?   Znum_key_value_groupsscalingrG   Z	is_causalZLinearrF   q_projk_projv_projo_projrI   rH   rN   r^   r_   r7   r7   rR   rM      s0   
zCohere2Attention.__init__r   position_embeddingsr   past_key_valuecache_positionrO   returnc                 K   s  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}| jd urGt|	|
||\}	}
|d ur||| j|d}|	|
|| j
|\}
}|d ur| jjdkr|j d }|
d d d d d |d d f |d d d d d |d d f }
}t}| jjdkr| jjdkr|dd	rtd
 nt| jj }|| |	|
||f| jsdn| j| j| jd|\}}|jg |dR   }| |}||fS )Nr`   r   )sincosrH   rl   flash_attention_2eagerZsdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r/   )Zdropoutrd   rH   )shaperJ   re   viewZ	transposerf   rg   rH   r   updater_   r^   _attn_implementationr   getloggerwarning_oncer   trainingrG   rd   Zreshape
contiguousrh   )rN   r   rj   r   rk   rl   rO   Zinput_shapeZhidden_shapeZquery_statesZ
key_statesZvalue_statesrp   ro   Zcache_kwargsseq_lenZattention_interfaceZattn_outputZattn_weightsr7   r7   rR   forward  sR   	

B	

zCohere2Attention.forwardN)NN)rS   rT   rU   rV   r   r   intrM   torchTensorr   r   
LongTensorr   r	   r~   r7   r7   r7   rR   r]      s(    r]   c                       s   e Zd Zdedef fddZeddd							dd
ejde	ejejf de
ej de
e de
e de
e de
ej dee de	eje
e	ejejf  f fddZ  ZS )Cohere2DecoderLayerr^   r_   c                    sB   t  || t||| _|| _|d | jj dk| _|j| _d S )Nr`   r   )rL   rM   r]   	self_attnr^   rI   
is_slidingrH   ri   rP   r7   rR   rM   W  s
   zCohere2DecoderLayer.__init__Zlast_cache_positionz4.53.0)versionNFr   rj   r   rk   rs   rC   rl   rO   rm   c              
   K   s<  | j rn|durnt|jd | j}	| jjdkr"|dd|	 df }nLt|jj	}
tj
tj|tjd| j d}t||
|}|d |	 d }tj|dd}tjt	|	|jd |jd	}||7 }|dddddd|f }|}| |}| jd|||||||d
|\}}| |}|| | }|f}|r||f7 }|S )ax  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
        Nr   rq   )dtype)Zdiagonalrn   r`   )mindevice)r   rj   r   rk   rs   rC   rl   r7   )r   maxrt   rH   r^   rw   r   Zfinfor   r   ZtrilZ	ones_likeboolwhereclamparanger   Zinput_layernormr   Zmlp)rN   r   rj   r   rk   rs   rC   rl   rO   Zeffective_seq_lenZ	min_dtypeZsliding_window_maskoffsetZmask_indexesZresidualZhidden_states_attentionZself_attn_weightsZhidden_states_mlpZoutputsr7   r7   rR   r~   ^  sF    



zCohere2DecoderLayer.forward)NNFFN)rS   rT   rU   r   r   rM   r   r   r   r   r   r   r   r   r   r	   FloatTensorr~   rW   r7   r7   rP   rR   r   V  s6    
	
r   c                   @   s   e Zd ZeZdS )Cohere2PreTrainedModelN)rS   rT   rU   r   Zconfig_classr7   r7   r7   rR   r     s    r   c                       s   e Zd Zdef fddZ									ddeej deej deej dee	 d	eej
 d
ee dee dee deej dee defddZ  ZS )Cohere2Modelr^   c                    s.   t  | t|j|jd| _t|d| _d S )N)r:   Zeps)r^   )rL   rM   r\   r:   rB   r"   rY   
rotary_embrN   r^   rP   r7   rR   rM     s   zCohere2Model.__init__Nr   r   position_idsr   r   rC   rs   output_hidden_statesrl   flash_attn_kwargsrm   c
              
   K   s  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|d u |d uA r*td| jr9| jr9|r9td d}|d u rB| 	|}|r]|d u r]| js]|j
\}}}t| j |||j| jd}|	d u ry|d uri| nd}tj|||j
d  |jd}	|d u r|	d}| |||	||}|}| ||}|rdnd }|rdnd }| jD ]%}|r||f7 }||f||||||	d	|
}|d }|r||d f7 }q| |}|r||f7 }t||||d
S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)Zmax_batch_sizeZmax_cache_lenr   r   r   r`   r   r7   )rj   r   rk   rs   rC   rl   )Zlast_hidden_stater   r   Z
attentions)r^   rs   r   rC   
ValueErrorZgradient_checkpointingr{   ry   rz   r    rt   r   r   r   Zget_seq_lengthr   r   Z	unsqueezeZ_update_causal_maskr   r!   r"   r
   )rN   r   r   r   r   r   rC   rs   r   rl   r   
batch_sizer}   _Zpast_seen_tokensZcausal_maskr   rj   Zall_hidden_statesZall_self_attnsZdecoder_layerZlayer_outputsr7   r7   rR   r~     s   






zCohere2Model.forward)	NNNNNNNNN)rS   rT   rU   r   rM   r   r   r   r   r   r   r   r   r	   r
   r~   rW   r7   r7   rP   rR   r     sD    	
r   c                       s:   e Zd Zdef fddZ							dddZ  ZS )	Cohere2ForCausalLMr^   c                    s   t  | d S r   )rL   rM   r   rP   r7   rR   rM   #  s   zCohere2ForCausalLM.__init__NTc	              	   K   s  |d ur2|d us|d |j d kr |d d |j d  d f }n|j d |j d kr2|d d |f }|d urb|d u rb| dd }||dkd |rb|d d |j d  d f }|jtjd}|d urr|d dkrr|d d}
n
|jtjdd d}
t|tr|j	dkr| j
jdks|
d	 d ur|
d	 j \}}}|
d	 j}n|
d
 j \}}|
d
 j}| jj||| | jjj|||d}|d ur||
d< |
|||||d |
S )Nrn   r`   r   )Zmemory_format)r   r   )r   r   r   rq   r   r   )sequence_lengthZtarget_lengthr   r   rl   r   logits_to_keep)r   rl   r   rC   r   )rt   longZcumsumZmasked_fill_cloner   Zcontiguous_format
isinstancer   ndimr^   rw   r   modelZ5_prepare_4d_causal_attention_mask_with_cache_positionZget_max_cache_shapeZlm_headweightr   rv   )rN   r   r   r   r   rl   r   rC   r   rO   Zmodel_inputsr   r   r   r   r7   r7   rR   prepare_inputs_for_generation&  sX   


	z0Cohere2ForCausalLM.prepare_inputs_for_generation)NNNNNTN)rS   rT   rU   r   rM   r   rW   r7   r7   rP   rR   r   "  s    r   )r   r   r   r   )3typingr   r   r   r   Ztorch.nnra   Ztorch.utils.checkpointZcache_utilsr   r   Zconfiguration_utilsr   Zmodeling_flash_attention_utilsr	   Zmodeling_outputsr
   Zmodeling_rope_utilsr   Zmodeling_utilsr   Zprocessing_utilsr   utilsr   Zutils.deprecationr   Zcohere.modeling_coherer   r   r   r   r   r   r   r   Zgemma2.modeling_gemma2r   Z
get_loggerrS   ry   r   rY   r\   rb   r]   r   r   r   r   __all__r7   r7   r7   rR   <module>   s4   (

 FZagX