o ZhS@sddlmZmZmZmZmZddlZddlmm Z ddlmZddl m Z ddl mZmZddlmZddlmZdd lmZdd lmZmZmZdd lmZmZdd lmZmZdd l m!Z!ddl"m#Z#m$Z$m%Z%ddl&m'Z'e$rddl(m)Z)ddl*m+Z+e%,e-Z.   d@deej/eej/dfdee0deej/deej/e0ffddZ1Gdddej2Z3e!4e3Gdddej2Z5ddZ6dAd d!Z7Gd"d#d#ej2Z8Gd$d%d%ej2Z9Gd&d'd'ej2Z:d(ej/d)e0dej/fd*d+Z;Gd,d-d-ej2Z< .dBd/ej2d0ej/d1ej/d2ej/deej/d3e=d4e=fd5d6Z>Gd7d8d8eZ?e#Gd9d:d:eZ@e#Gd;d<dd>e@eZBgd?ZCdS)C)CallableListOptionalTupleUnionN)nn)ACT2FN)Cache DynamicCache)GenerationMixin)AttentionMaskConverter)GradientCheckpointingLayer)BaseModelOutputWithPastMoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)ALL_LAYERNORM_LAYERS)auto_docstringis_torch_flex_attn_availablelogging)GraniteMoeConfig) BlockMask)make_flex_block_causal_mask gate_logits num_expertsattention_maskreturncs|dus t|ts dSt|tr#|djtjfdd|Ddd}tjjj|dd}tj||dd\}}tjj ||}|durStj | dd} tj |dd} ng|j \} } |j d| | } |dddddddf | | | ||fd||}tj| |ddtj|dd} |ddddddf | | | |fd|}tj||ddtj|dd} t| | d}||S)a Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between experts is too unbalanced. Args: gate_logits: Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of shape [batch_size X sequence_length, num_experts]. num_experts: Number of experts top_k: The number of experts to route per-token, can be also interpreted as the `top-k` routing parameter. attention_mask (`torch.Tensor`, *optional*): The attention_mask used in forward function shape [batch_size X sequence_length] if not None. Returns: The auxiliary loss. Nrcsg|]}|qS)to).0Z layer_gateZcompute_devicer#a/var/www/auris/lib/python3.10/site-packages/transformers/models/granitemoe/modeling_granitemoe.py Oz,load_balancing_loss_func..dim) isinstancetupledevicetorchcatr functionalsoftmaxtopkZone_hotmeanfloatshapeexpandreshaper$sum unsqueeze)rr top_kr!Zconcatenated_gate_logitsZrouting_weights_Zselected_expertsZ expert_maskZtokens_per_expertZrouter_prob_per_expert batch_sizesequence_lengthnum_hidden_layersZexpert_attention_maskZ router_per_expert_attention_maskZ overall_lossr#r&r'load_balancing_loss_func-s>    rAcs.eZdZdfdd ZddZddZZS) GraniteMoeRMSNormư>cs&ttt||_||_dS)z@ GraniteMoeRMSNorm is equivalent to T5LayerNorm N)super__init__r Parameterr0Zonesweightvariance_epsilon)self hidden_sizeeps __class__r#r'rEs  zGraniteMoeRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)Nrr,T)Zkeepdim) dtyper$r0float32powr5ZrsqrtrHrG)rI hidden_statesZ input_dtypeZvariancer#r#r'forwards  zGraniteMoeRMSNorm.forwardcCst|jjd|jS)Nz, eps=)r.rGr7rHrIr#r#r' extra_reprszGraniteMoeRMSNorm.extra_repr)rC)__name__ __module__ __qualname__rErRrT __classcell__r#r#rLr'rBsrBcs8eZdZddeffdd ZeeddZZ S)GraniteMoeRotaryEmbeddingNconfigcstt|dr|jdur|jd|jd|_nd|_|j|_|j|_||_ t |j|_ | |j |\}|_ |j d|dd|j|_dS)N rope_scaling rope_typetypedefaultinv_freqF) persistent)rDrEhasattrr[getr\max_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenrZrZ rope_init_fnattention_scalingZregister_bufferr_Zoriginal_inv_freq)rIrZr/r_rLr#r'rEs   z"GraniteMoeRotaryEmbedding.__init__c Cs|jddddf|jddd|j}|dddddf}t|jjtr6|jjdkr6|jjnd}t j |dd+|| dd}t j ||fdd }| |j}||j} Wdn1smwY|j|jd | j|jd fS) Nrr,rZmpscpuF) device_typeenabledrr*)rN)r_r6r8r7r$r/r-r]strr0Zautocast transposer1cosrdsinrN) rIx position_idsZinv_freq_expandedZposition_ids_expandedrfZfreqsZembrjrkr#r#r'rRs0&z!GraniteMoeRotaryEmbedding.forwardN) rUrVrWrrEr0Zno_gradrrRrXr#r#rLr'rYs rYcCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Nr,rr*)r7r0r1)rlx1Zx2r#r#r' rotate_halfsrpcCsD||}||}||t||}||t||}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. )r;rp)qkrjrkrmZ unsqueeze_dimZq_embedZk_embedr#r#r'apply_rotary_pos_embs  rscs6eZdZdedededdffdd Zdd ZZS) GraniteMoeParallelExpertsr input_size output_sizer"Ncs6ttt||||_||_||_||_ dS)a Initialize the GraniteMoeParallelExperts module. The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py) used in vllm. Args: num_experts (int): Number of experts. input_size (int): Size of the input. output_size (int): Size of the output. N) rDrErrFr0emptyrGr rurv)rIr rurvrLr#r'rEs  z"GraniteMoeParallelExperts.__init__cCsP|j|dd}g}t|jD]}|t|||j|qtj|dd}|S)a Forward pass of the GraniteMoeParallelExperts module. Args: inputs (Tensor): Input tensor. expert_size: Expert size information. Returns: Tensor: Output tensor. rr*) splitranger appendFZlinearrGr0r1)rIZinputs expert_sizeZ input_listZ output_listiresultsr#r#r'rRs  z!GraniteMoeParallelExperts.forwardrUrVrWintrErRrXr#r#rLr'rtsrtcs2eZdZdededeffdd ZddZZS)GraniteMoeTopKGatingrur r<cs2t||_||_||_tj||dd|_dS)a Initialize the top-k gating mechanism. Args: input_size (`int`): Size of the input. num_experts (`int`): Number of experts. top_k (`int`): Number of top experts to select. FbiasN)rDrEr rur<rLinearlayer)rIrur r<rLr#r'rEs zGraniteMoeTopKGating.__init__cCs||}|j|jdd\}}tj|dd|}tj|d|j g|j |j d}| d|d}| d}|}|} | d\} } | j|jdd} |}|| } | | | ||fS)Nrr*rrNr/trunc)Z rounding_mode)rr6r4r<r0r3Ztype_aszerossizer rNr/Zscatterlongr:tolistflattensortdiv)rIrQlogitsZ top_k_logitsZ top_k_indicesZ top_k_gatesrZgatesr|Z top_k_expertsr=Zindex_sorted_experts batch_index batch_gatesr#r#r'rR!szGraniteMoeTopKGating.forwardrr#r#rLr'r srcs.eZdZdZdeffdd ZddZZS) GraniteMoeMoEz A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts. Args: config: Configuration object with model hyperparameters. rZcsptt||j|_|j|_t|j|_t |j |j|jd|_ t |j |j|j|_ t |j|j |jd|_dS)Nr)rur r<)rDrrErJruZintermediate_sizer Z hidden_act activationrtnum_local_experts input_linear output_linearrnum_experts_per_tokrouterrIrZrLr#r'rEFs  zGraniteMoeMoE.__init__cCs|\}}}|d|}||\}}}}} ||} || |} | jddd} || d| d} || |} | |dddf} tj|||j f| j | j d}| d|| }| |||j }|| fS)a Forward pass of the mixture of experts layer. Args: layer_input (Tensor): Input tensor. Returns: Tensor: Output tensor. Tensor: Router logits. r,rr*rrNr)rr9rrchunkrrr0rrurNr/Z index_addview)rIZ layer_inputbszlengthZemb_sizer=rrr| router_logitsZ expert_inputsrQZchunked_hidden_statesZexpert_outputsrZ layer_outputr#r#r'rRUs   zGraniteMoeMoE.forward)rUrVrW__doc__rrErRrXr#r#rLr'r=srrQn_repcCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)r7r8r9)rQrbatchnum_key_value_headsslenhead_dimr#r#r' repeat_kvvs 0rcseZdZdZddedeeffdd Z      ddej d eej d eej d ee d e d eej dee ej ej fde ej eej ee ej ffddZZS)GraniteMoeAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNrZ layer_idxcs t||_||_|durtd|jjd|j|_|j |_ |j |_ |j |j |_ |j |_ |j |j |_d|_|j|_|j |j |j krUtd|j d|j dtj|j |j |j |jd|_tj|j |j |j |jd|_tj|j |j |j |jd|_tj|j |j |jd|_dS)NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r)rDrErZrlogger warning_oncerMrUattention_dropoutrJnum_attention_heads num_headsrrnum_key_value_groupsZ is_causalZattention_multiplierscaling ValueErrorrrZattention_biasq_projk_projv_projo_projrIrZrrLr#r'rEs2  zGraniteMoeAttention.__init__FrQr!rmpast_key_value use_cachecache_positionposition_embeddingsr"cKsj|\} } } ||} ||} ||}| | | |j|jdd} | | | |j|jdd} || | |j|jdd}|durF|nd\}}|durWt | | ||\} } |durl|||d}| | ||j |\} }t }|j jdkr|j jdkr|ddrtd nt|j j}||| | ||f|jsd n|j|jd |\}}|| | d }||}|||fS) Nrr)NN)rkrjreagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)dropoutrr,)rrrrrrrrirrsupdatereager_attention_forwardrZ_attn_implementationrbrrrtrainingrrr)rIrQr!rmrrrrkwargsrZq_lenr=Z query_states key_states value_statesrjrkZ cache_kwargsZattention_interface attn_output attn_weightsr#r#r'rRsF         zGraniteMoeAttention.forwardrn)NNNFNN)rUrVrWrrrrrEr0Tensor LongTensorr boolrrRrXr#r#rLr'rs4# rrmodulequerykeyvaluerrc Kst||j}t||j} t||dd|} |dur3|ddddddd|jdf} | | } tjj| dtj d |j } tjj | ||j d} t| | } | dd} | | fS)Nrrr,)r+rN)prr)rrr0matmulrir7rr2r3rOr$rNrr contiguous) rrrrr!rrrrrr causal_maskrr#r#r'rs & rcseZdZdedeffdd Z        ddejdeejd eej d ee d ee d ee d eej dee dee ejejfde ej ee ej ej fffddZZS)GraniteMoeDecoderLayerrZrcsZt|j|_t||d|_t||_t|j|jd|_ t|j|jd|_ |j |_ dS)N)rZrrK) rDrErJr self_attnrblock_sparse_moerB rms_norm_epsinput_layernormpost_attention_layernormresidual_multiplierrrLr#r'rEs   zGraniteMoeDecoderLayer.__init__NFrQr!rmrrrroutput_router_logitsrr"c  Ks|} ||}|jd|||||||| d| \}} } | ||j}|} ||}||\}}| ||j}|f}|rA|| f7}|rH|| f7}|rO||f7}|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, query_sequence_length, key_sequence_length)` if default attention is used. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence output_router_logits (`bool`, *optional*): Whether or not to return the logits of all the routers. They are useful for computing the router loss, and should not be returned during inference. position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model )rQr!rmrrrrrNr#)rrrrr)rIrQr!rmrrrrrrrZresidualZself_attn_weightsZpresent_key_valueroutputsr#r#r'rRs6&       zGraniteMoeDecoderLayer.forward)NNNFFNFN)rUrVrWrrrEr0rrrr rr FloatTensorrRrXr#r#rLr'rs>   rc@s@eZdZeZdZdZdgZdgZdZ dZ dZ dZ dZ ddZdS) GraniteMoePreTrainedModelmodelTrpast_key_valuesFcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|t rQ|jj ddSt|trc|jjjd|jjddSdS)Nr)r5Zstdg?)r-rrrGdataZnormal_rZZinitializer_rangerZzero_ Embedding padding_idxrBZfill_rt)rIrr#r#r' _init_weightsbs      z'GraniteMoePreTrainedModel._init_weightsN)rUrVrWrZ config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementZ_supports_flash_attn_2Z_supports_sdpaZ_supports_cache_classZ_supports_quantized_cacheZ_supports_static_cacherr#r#r#r'rUs rcs0eZdZdeffdd ZddZddZe           d"d ee j d ee j d ee j d ee e ee jfd ee jdeedeedeedeedeedee j de eeffddZ d#d e e j dfde j de j d e def ddZed e j dedede jde j def d d!ZZS)$GraniteMoeModelrZcstj|_j|_tjj|j|_t fddt j D|_ t jjd|_d|_j|_j|_j|_|j|j|_j|_j|_j|_|jdkr]tnd|_|dS)Ncsg|]}t|qSr#)r)r%rrZr#r'r(zr)z,GraniteMoeModel.__init__..rFZrope)rDrEZ pad_token_idr vocab_sizerrrJ embed_tokensZ ModuleListryr@layersrBrnormgradient_checkpointingembedding_multiplierrrrrcZ rope_thetaZposition_embedding_typerY rotary_emb post_initrrLrr'rEss$  zGraniteMoeModel.__init__cC|jSrnrrSr#r#r'get_input_embeddingsz$GraniteMoeModel.get_input_embeddingscC ||_dSrnrrIrr#r#r'set_input_embeddings z$GraniteMoeModel.set_input_embeddingsN input_idsr!rmr inputs_embedsrroutput_hidden_statesr return_dictrr"c  CsH|dur|n|jj}|dur|n|jj}|dur|n|jj}| dur$| n|jj} |du|duAr4td|jrC|jrC|rCt dd}|durL| |}||j }d} |rft |t sfd} t|}t d| dur|durr|nd} tj| | |jd|jd} |dur| d}|||| ||}|}d}|jdur|||}|rd nd}|rd nd}| rd nd}d}|jD]6}|r||f7}|||||||| | |d }|d}|r||rd nd}|r||df7}| r||d f7}q||}|r||f7}|r|nd}| r |}| std d||||fDSt|||||dS)Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FTzWe detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)rrr/r#)r!rmrrrrrrrr,css|] }|dur|VqdSrnr#)r%vr#r#r' sz*GraniteMoeModel.forward..)Zlast_hidden_staterrQ attentionsr)rZrrruse_return_dictrrrrrrrr-r r Zfrom_legacy_cacheget_seq_lengthr0aranger7r/r;_update_causal_maskrrrZto_legacy_cacher.r)rIrr!rmrrrrrrrrZreturn_legacy_cachepast_seen_tokensrrQrZall_hidden_statesZall_self_attnsZall_router_logitsZnext_decoder_cacheZ decoder_layerZ layer_outputsZ next_cacher#r#r'rRs               zGraniteMoeModel.forwardFr input_tensorc Cs:|jjdkr|dur|dkr|SdS|jjdkr&t|tjr$t|}|S|dur.|nd}|dur7|jnd}|jjdkrO|sO|sOt j ||||j drOdS|j }|j d} |r^|} nt|tjri|j d n|| d} |j|| | |||j dd } |jjdkr|dur|jjd vr|st|j} t | | } | S) NZflash_attention_2rZflex_attentionrFr)rZpast_key_values_lengthZ is_trainingrr,)r? target_lengthrNrr>)cudaZxpuZnpu)rZranyr-r0rrrZis_compileabler Z_ignore_causal_mask_sdparrNr7Zget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr/r]finfominZ_unmask_unattended) rIr!rrrrrZusing_compilable_cacherNr?rr min_dtyper#r#r'r sT           z#GraniteMoeModel._update_causal_maskr?rrNr>c KsD|dur|dkr|}|St|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9}|ddddddf |ddd}|dur| }|j d} |ddddddd| f|ddddddf |j} | dk} |ddddddd| f | ||ddddddd| f<|S) aM Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. N)Z fill_valuerNr/r)Zdiagonalrr,r)r+r0r r fullr/Ztriurr9r8cloner7r$Z masked_fill) r!r?rrNrr>rrr Z mask_lengthZ padding_maskr#r#r'rMs,  $ 6  zEGraniteMoeModel._prepare_4d_causal_attention_mask_with_cache_position) NNNNNNNNNNN)F)rUrVrWrrErrrrr0rrrr rrrrrrRr staticmethodrrNrrXr#r#rLr'rqs     |  Drc s"eZdZdgZdeffdd ZddZddZd d Zd d Z d dZ ddZ e             d%de ejde ejde ejde eeeejfde ejde ejde ede ede ede ede ede ejdeeejfd eeeffd!d"Zed#d$ZZS)&GraniteMoeForCausalLMzlm_head.weightrZcsXt|t||_|j|_tj|j|jdd|_|j |_ |j |_ |j |_ | dS)NFr)rDrErrrrrrJlm_headrouter_aux_loss_coefrr rrrrLr#r'rEs   zGraniteMoeForCausalLM.__init__cCs|jjSrnrrrSr#r#r'rsz*GraniteMoeForCausalLM.get_input_embeddingscCs ||j_dSrnrrr#r#r'rs z*GraniteMoeForCausalLM.set_input_embeddingscCrrnrrSr#r#r'get_output_embeddingsrz+GraniteMoeForCausalLM.get_output_embeddingscCrrnr)rIZnew_embeddingsr#r#r'set_output_embeddingsrz+GraniteMoeForCausalLM.set_output_embeddingscCrrnr)rIdecoderr#r#r' set_decoderrz!GraniteMoeForCausalLM.set_decodercCrrnrrSr#r#r' get_decoderrz!GraniteMoeForCausalLM.get_decoderNrrr!rmrrlabelsrrrrrrlogits_to_keepr"c Ks|dur|n|jj}| dur| n|jj} | dur| n|jj} | dur$| n|jj} |j|||||||| | | | d }|d}t| trGt| dn| }| |dd|ddf}||jj }d}|duru| }|j ||fd|jj i|}d}| rt| r|jn|d|j|j|}|dur||j||j7}| s|f|dd}| r|f|}|dur|f|S|St||||j|j|j|jdS)al labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, GraniteMoeForCausalLM >>> model = GraniteMoeForCausalLM.from_pretrained("ibm/PowerMoE-3b") >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rr!rmrrrrrrrrrrr,r)lossaux_lossrrrQrr)rZrrrrrr-rslicerZlogits_scalingr6Z loss_functionrrArr rrr$r/rrrQr)rIrr!rmrrrrrrrrrrrrrQZ slice_indicesrrroutputr#r#r'rRst(  zGraniteMoeForCausalLM.forwardcs.d}|D]}|tfdd|Df7}q|S)Nr#c3s$|] }|d|jVqdS)rN)Z index_selectr$r/)r%Z past_statebeam_idxr#r'rs"z7GraniteMoeForCausalLM._reorder_cache..)r.)rr"Zreordered_pastZ layer_pastr#r!r'_reorder_caches z$GraniteMoeForCausalLM._reorder_cache) NNNNNNNNNNNNr)rUrVrWZ_tied_weights_keysrrErrrrrrrrr0rrrr rrrrrrrRrr#rXr#r#rLr'rsl         lr)rrr)NrN)Nr)r)Dtypingrrrrrr0Ztorch.nn.functionalrr2r{Z activationsr Z cache_utilsr r Z generationr Zmodeling_attn_mask_utilsr Zmodeling_layersrZmodeling_outputsrrrZmodeling_rope_utilsrrZmodeling_utilsrrZ pytorch_utilsrutilsrrrZconfiguration_granitemoerZ!torch.nn.attention.flex_attentionrZintegrations.flex_attentionrZ get_loggerrUrrrrAModulerBrzrYrprsrtrrrrr6rrrrr__all__r#r#r#r's            S # .09c Z