o ZhD@sdZddlmZmZmZmZmZddlZddlZddlm Z ddl m Z ddl m Z ddlmZdd lmZmZdd lmZdd lmZdd lmZdd lmZddlmZmZddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'e(e)Z*Gddde j+Z,e-e,Gddde$Z.ddZ/d*ddZ0Gddde!Z1GdddeZ2Gdd d eZ3Gd!d"d"e#Z4Gd#d$d$e"Z5Gd%d&d&e eZ6Gd'd(d(e Z7gd)Z8dS)+zPyTorch Cohere model.)CallableListOptionalTupleUnionN)nn)Cache)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)dynamic_rope_update)ALL_ATTENTION_FUNCTIONS)Unpack)ALL_LAYERNORM_LAYERS) LossKwargslogging)LlamaAttentionLlamaForCausalLMLlamaMLP LlamaModelLlamaPreTrainedModelLlamaRotaryEmbeddingeager_attention_forward) CohereConfigcs&eZdZdfdd ZddZZS) CohereLayerNormNh㈵>Fcs&ttt||_||_dS)zcThe hidden size can be a tuple or an int. The tuple is used for QKNorm to normalize across head_dimN)super__init__r ParametertorchZonesweightvariance_epsilon)self hidden_sizeepsbias __class__X/var/www/auris/lib/python3.10/site-packages/transformers/models/cohere/modular_cohere.pyr!8s  zCohereLayerNorm.__init__cCsl|j}|tj}|jddd}||djddd}||t||j}|jtj|}||S)NT)Zkeepdimr) dtypetor#Zfloat32meanpowZrsqrtr%r$)r& hidden_statesZ input_dtyper1Zvariancer,r,r-forward>s  zCohereLayerNorm.forward)NrF)__name__ __module__ __qualname__r!r4 __classcell__r,r,r*r-r7src@s eZdZeeddZdS)CohereRotaryEmbeddingc Cs|jddddf|jddd}|dddddf}t|jjtr2|jjdkr2|jjnd}tj |dd*|| dd}tj |ddd }| |j }||j } Wdn1shwY|j|jd | j|jd fS) Nrr.rZmpscpuF) device_typeenabledrdimr/)Zinv_freqfloatexpandshape isinstanceZdevicetypestrr#Zautocast transposeZrepeat_interleavecosZattention_scalingsinr0r/) r&x position_idsZinv_freq_expandedZposition_ids_expandedr;ZfreqsZembrGrHr,r,r-r4Ls(&zCohereRotaryEmbedding.forwardN)r5r6r7r#Zno_gradrr4r,r,r,r-r9Ksr9cCsB|ddddf}|ddddf}tj| |gddd}|S)N.rrr.r=)r#stackflatten)rIx1Zx2Zrot_xr,r,r- rotate_half\srOc Csj|j}|}|}||}||}||t||}||t||}|j|d|j|dfS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. r?)r/r@Z unsqueezerOr0) qkrGrHrJZ unsqueeze_dimr/Zq_embedZk_embedr,r,r-apply_rotary_pos_embds  rRcseZdZfddZZS) CohereMLPcsRt|tj|j|jdd|_tj|j|jdd|_tj|j|jdd|_dS)NF)r)) r r!rLinearr'Zintermediate_sizeZ gate_projZup_projZ down_projr&configr*r,r-r!s zCohereMLP.__init__)r5r6r7r!r8r,r,r*r-rSsrScseZdZdZddedeeffdd Z  ddej de ej ej fd eej d ee d eej d e ed e ej eej ee ej ffddZZS)CohereAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNrV layer_idxcsTt|||j|_|jr(t|j|jf|jd|_t|j|jf|jd|_ dSdS)Nr'r() r r! use_qk_normrZnum_attention_headshead_dimlayer_norm_epsq_normZnum_key_value_headsk_normr&rVrXr*r,r-r!s zCohereAttention.__init__r3position_embeddingsattention_maskpast_key_valuecache_positionkwargsreturncKsn|jdd}g|d|jR}|||} |||} |||} |jr6|| } || } | dd} | dd} | dd} |\} } t | | | | \} } |durj| | |d}| | | |j |\} } t }|jjdkr|jjdkr|ddrtd nt|jj}||| | | |f|jsd n|j|jd |\}}|jg|dR}||}||fS) Nr.rr)rHrGrceagerZsdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)Zdropoutscaling)rBr[Zq_projviewZk_projZv_projrZr]r^rFrRupdaterXrrVZ_attn_implementationgetloggerZ warning_oncerZtrainingZattention_dropoutriZreshape contiguousZo_proj)r&r3r`rarbrcrdZ input_shapeZ hidden_shapeZ query_statesZ key_statesZ value_statesrGrHZ cache_kwargsZattention_interfaceZ attn_outputZ attn_weightsr,r,r-r4sL           zCohereAttention.forwardN)NN)r5r6r7__doc__rrintr!r#Tensorrr LongTensorrr r4r8r,r,r*r-rWs(rWcseZdZdedeffdd Z       ddejdeejd eej d ee d ee d ee d eej dee ejejfde ede ejee ejejfffddZZS)CohereDecoderLayerrVrXcs@t|j|_t||d|_t||_t|j|jd|_ dS)N)rVrXrY) r r!r'rW self_attnrSmlprr\input_layernormr_r*r,r-r!s  zCohereDecoderLayer.__init__NFr3rarJrbrg use_cachercr`rdrec  Ksb|} ||}|jd||||||||d| \} } ||} | | | }|f}|r/|| f7}|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, query_sequence_length, key_sequence_length)` if default attention is used. past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. )r3rarJrbrgrxrcr`Nr,)rwrurv)r&r3rarJrbrgrxrcr`rdZresidualZhidden_states_attentionZself_attn_weightsZhidden_states_mlpoutputsr,r,r-r4s(   zCohereDecoderLayer.forward)NNNFFNN)r5r6r7rrqr!r#rrrrsr boolrrr FloatTensorr4r8r,r,r*r-rts<    rtc@seZdZddZdS)CoherePreTrainedModelcCs|jj}t|tjr"|jjjd|d|jdur |jj dSdSt|tj rC|jjjd|d|j durA|jj|j  dSdSt|t rQ|jj ddSdS)Nrh)r1stdg?)rVZinitializer_rangerCrrTr$dataZnormal_r)Zzero_Z EmbeddingZ padding_idxrZfill_)r&moduler}r,r,r- _init_weightss     z#CoherePreTrainedModel._init_weightsN)r5r6r7rr,r,r,r-r|s r|cs"eZdZdeffdd ZZS) CohereModelrVcsNttfddtjD|_td|_t j j d|_ dS)Ncsg|]}t|qSr,)rt).0rXrVr,r- +sz(CohereModel.__init__..rrY) r r!rZ ModuleListrangeZnum_hidden_layersZlayersr9Z rotary_embrr'r\ZnormrUr*rr-r!(s  zCohereModel.__init__)r5r6r7rr!r8r,r,r*r-r'src@s eZdZdS)KwargsForCausalLMN)r5r6r7r,r,r,r-r1s rcseZdZfddZ           ddeejdeejdeejdeee e ej fd eej d eejd ee d ee d ee deejdee ejfdeedefddZZS)CohereForCausalLMcs*t|t||_|j|_|j|_dSro)r r!rmodel logit_scaleZtie_word_embeddingsrUr*r,r-r!5s   zCohereForCausalLM.__init__Nr input_idsrarJpast_key_values inputs_embedslabelsrxrgoutput_hidden_statesrclogits_to_keeprdrec  Ks|dur|n|jj}| dur| n|jj} |jd|||||||| | d | } | j}t| tr4t| dn| }||dd|ddf}||j }d}|dur]|j d|||jj d| }t ||| j | j| jdS)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >> from transformers import AutoTokenizer, CohereForCausalLM >> model = CohereForCausalLM.from_pretrained("CohereForAI/c4ai-command-r-v01") >> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01") >> prompt = "Hey, are you conscious? Can you talk to me?" >> inputs = tokenizer(prompt, return_tensors="pt") >> # Generate >> generate_ids = model.generate(inputs.input_ids, max_length=30) >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rrarJrrrxrgrrc)logitsr vocab_size)lossrrr3 attentionsr,)rVrgrrZlast_hidden_staterCrqsliceZlm_headrZ loss_functionrr rr3r)r&rrarJrrrrxrgrrcrrdryr3Z slice_indicesrrr,r,r-r4;s<%   zCohereForCausalLM.forward) NNNNNNNNNNr)r5r6r7r!rr#rsrrrr rr{rzrqrrr r4r8r,r,r*r-r4sP      r)rrr|)Nr)9rptypingrrrrrr#Ztorch.utils.checkpointrZ cache_utilsr Zmodeling_flash_attention_utilsr Zmodeling_layersr Zmodeling_outputsr r Zmodeling_rope_utilsrZmodeling_utilsrZprocessing_utilsrZ pytorch_utilsrutilsrrZllama.modeling_llamarrrrrrrZconfiguration_cohererZ get_loggerr5rmModulerappendr9rOrRrSrWrtr|rrr__all__r,r,r,r-s<        $   IE  R