o Zhl  @sdZddlZddlmZmZmZmZmZddlZddl m m Z ddl Zddlm Z ddlmZddlmZmZddlmZdd lmZdd lmZmZdd lmZmZmZdd lmZdd l m!Z!m"Z"m#Z#ddl$m%Z%m&Z&ddl'm(Z(er~ddlm)Z)e&rddl*m+Z+m,Z,ddl-m.Z.nd\Z.Z,Z+e%rddl/m0Z0m1Z1nd\Z1Z0e2e.e,e0e1e+fZ3e#4e5Z6   dCdeej7eej7dfdee8deej7deej7e8ffddZ9Gddde j:Z;d ej7d!e8dej7fd"d#ZGd(d)d)e>Z?Gd*d+d+e>Z@e>e?e@d,ZAGd-d.d.e j:ZBGd/d0d0e j:ZCGd1d2d2e j:ZDGd3d4d4e j:ZEGd5d6d6e j:ZFe!Gd7d8d8eZGeEeFd9ZHe!Gd:d;d;eGZIGdd?Gd@dAdAeGZKgdBZLdS)DzPyTorch Jamba model.N)AnyDictOptionalTupleUnion)nn)ACT2FN)Cache DynamicCache)GenerationMixin)AttentionMaskConverter)!flash_attn_supports_top_left_maskis_flash_attn_available)MoeCausalLMOutputWithPastMoeModelOutputWithPast SequenceClassifierOutputWithPast)PreTrainedModel)auto_docstringcan_return_tuplelogging)is_causal_conv1d_availableis_mamba_ssm_available) JambaConfig)_flash_attention_forward)mamba_inner_fnselective_scan_fn)selective_state_update)NNN)causal_conv1d_fncausal_conv1d_updateNN router_logits num_expertsattention_maskreturncs|dus t|ts dSt|tr#|djtjfdd|Ddd}tjjj|dd}tj||dd\}}tjj ||}|durStj | dd} tj |dd} ng|j \} } |j d| | } |dddddddf | | | ||fd||}tj| |ddtj|dd} |ddddddf | | | |fd|}tj||ddtj|dd} t| | d}||S)a Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between experts is too unbalanced. Args: router_logits: Logits from the `router`, should be a tuple of model.config.num_hidden_layers tensors of shape [batch_size X sequence_length, num_experts]. num_experts: Number of experts top_k: The number of experts to route per-token, can be also interpreted as the `top-k` routing parameter. attention_mask (`torch.Tensor`, *optional*): The attention_mask used in forward function shape [batch_size X sequence_length] if not None. Returns: The auxiliary loss. Nrcsg|]}|qS)to).0Z layer_routerZcompute_devicer'W/var/www/auris/lib/python3.10/site-packages/transformers/models/jamba/modeling_jamba.py esz,load_balancing_loss_func..dim) isinstancetupledevicetorchcatr functionalsoftmaxtopkone_hotmeanfloatshapeexpandreshaper(sum unsqueeze)r#r$top_kr%Zconcatenated_router_logitsrouting_weights_selected_experts expert_maskZtokens_per_expertZrouter_prob_per_expert batch_sizesequence_lengthnum_hidden_layersZexpert_attention_maskZ router_per_expert_attention_maskZ overall_lossr'r*r+load_balancing_loss_funcBsB    rHcs.eZdZdfdd ZddZddZZS) JambaRMSNormư>cs&ttt||_||_dS)z; JambaRMSNorm is equivalent to T5LayerNorm N)super__init__r Parameterr3onesweightvariance_epsilon)self hidden_sizeeps __class__r'r+rLs  zJambaRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)Nr"r/T)Zkeepdim) dtyper(r3float32powr9ZrsqrtrPrO)rQ hidden_states input_dtypeZvariancer'r'r+forwards  zJambaRMSNorm.forwardcCst|jjd|jS)Nz, eps=)r1rOr;rPrQr'r'r+ extra_reprszJambaRMSNorm.extra_repr)rJ)__name__ __module__ __qualname__rLr[r] __classcell__r'r'rTr+rIsrIrYn_repcCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)r;r<r=)rYrbbatchnum_key_value_headsslenhead_dimr'r'r+ repeat_kvs 0rgc seZdZdZejdffdd Z ddejdejdede e e e fd e ejejff d d Zd ejfd dZdde ed efddZd e e eje ejffddZedde e e ejd dfddZZS) HybridMambaAttentionDynamicCachea A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache (which has a constant shape regardless of seq_len). This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states` and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`, while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors). For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors), while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`, and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`. Nc s0t||_|j|_d|_|j|j}|j}|j}g|_ g|_ g|_ t |j D]P}|j|dkrS|j tj|||dg7_ |j tj|||dg7_ q)|j tjggdg7_ |j tjggdg7_ |j |q)fddt |j D|_fddt |j D|_dS)NFmambar2rVr2c g|] }tjggdqSrkr3tensorr)rBrEr2r'r+r, z=HybridMambaAttentionDynamicCache.__init__..crlrmrnrprqr'r+r,rr)rKrLrVlayers_block_typehas_previous_state mamba_expandrR mamba_d_state mamba_d_conv conv_states ssm_statestransformer_layersrangerGr3zerosroappend key_cache value_cache) rQconfigrErVr2intermediate_sizessm_state_sizeconv_kernel_sizeirTrqr+rLs.     z)HybridMambaAttentionDynamicCache.__init__ key_states value_states layer_idx cache_kwargsr&cCsz|j|jddkr||j|<||j|<ntj|j||gdd|j|<tj|j||gdd|j|<|j||j|fS)Nr/rr"r-)r~r;rr3r4)rQrrrrr'r'r+updates   z'HybridMambaAttentionDynamicCache.updatebeam_idxcCstt|jD]V}|j|j}|j|d|||j|<|j|j}|j|d|||j|<|j|j}|j|d|||j|<|j|j}|j|d|||j|<qdS)zDReorders the cache for beam search, given the selected beam indices.rN) r{lenr~r2Z index_selectr(rrxry)rQrrr2r'r'r+ reorder_caches     z.HybridMambaAttentionDynamicCache.reorder_cachercCs:||jvr |jdn|}t|j|krdS|j|jdS)zYReturns the sequence length of the cached states. A layer index can be optionally passed.r)rzrr~r;)rQrr'r'r+get_seq_lengthsz/HybridMambaAttentionDynamicCache.get_seq_lengthcCtdNzIHybridMambaAttentionDynamicCache does not have a legacy cache equivalent.NotImplementedErrorr\r'r'r+to_legacy_cachez0HybridMambaAttentionDynamicCache.to_legacy_cachepast_key_valuesr cCrrr)clsrr'r'r+from_legacy_cache sz2HybridMambaAttentionDynamicCache.from_legacy_cacheN)r)r^r_r`__doc__r3Zfloat16rLTensorintrrstrrrr LongTensorrrr classmethod FloatTensorrrar'r'rTr+rhs(    ",rhcseZdZdZddedeeffdd Z      ddej d eej d eej d ee d e d e deej de ej eej ee ej ffddZZS)JambaAttentionz Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer and "Generating Long Sequences with Sparse Transformers". Nrrcst||_||_|durtd|jjd|j|_|j |_ |j|j |_ |j |_ |j |j |_ d|_|j|_|j |j |jkrQtd|jd|j dtj|j|j |j dd|_tj|j|j |j dd|_tj|j|j |j dd|_tj|j |j |jdd|_dS) NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).Fbias)rKrLrrlogger warning_oncerUr^rRZnum_attention_heads num_headsrfrdnum_key_value_groups is_causalattention_dropout ValueErrorrLinearq_projk_projv_projo_proj)rQrrrTr'r+rLs0   zJambaAttention.__init__FrYr% position_idspast_key_valueoutput_attentions use_cachecache_positionr&cCs|\}} } ||} ||} ||} | || |j|jdd} | || |j|jdd} | || |j|jdd} |durN| | | |j \} } t | |j } t | |j } t | | ddt|j}|dur|ddddddd| jdf}||}tjj|dt jd| j}tjj||j|jd}t || }|||j| |jfkrtd||j| |jfd ||dd}||| |j}||}|sd}|||fS) Nrr"rrr/r.rV)ptrainingz `attn_output` should be of size z , but is ) sizerrrviewrrf transposerdrrrgrr3matmulmathsqrtr;rr5r6rWr(rVdropoutrrr contiguousr=rRr)rQrYr%rrrrrbszq_lenrB query_statesrr attn_weights causal_mask attn_outputr'r'r+r[3s<      &   zJambaAttention.forwardrNNNFFN)r^r_r`rrrrrLr3rrrhboolrr[rar'r'rTr+rs4  rcsleZdZdZfddZ      ddejdeejdeejd ee d e d e d eejfd dZ Z S)JambaFlashAttention2aF Jamba flash attention module. This module inherits from `JambaAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. cstj|i|t|_dSr)rKrLr_flash_attn_uses_top_left_mask)rQargskwargsrTr'r+rLrs zJambaFlashAttention2.__init__NFrYr%rrrrrc Ks|\} } } ||} ||} ||}| | | |j|j} | | | |j|jdd} || | |j|jdd}|durJ| | ||j \} }t | |j } t ||j }|j s[dn|j}| j}|tjkrtrot}nt|jdrz|jj}n|jjj}td|d| |} | |} ||}| dd} |dd}t| | ||| |t|jdd|j|jd }|| | |j !}|"|}|sd}|||fS) Nrr"_pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .sliding_window)rrrZuse_top_left_mask)#rrrrrrrfrdrrrrgrrrrVr3rWZis_autocast_enabledZget_autocast_gpu_dtypehasattrrrrOrrr(rgetattrrrr=rRrr)rQrYr%rrrrrrrrrBrrrZ dropout_raterZZ target_dtyperrr'r'r+r[zsZ                  zJambaFlashAttention2.forwardr) r^r_r`rrLr3rrrrhrr[rar'r'rTr+rks0  rcseZdZdZ      ddejdeejdeejdeede d e d eejd e ejeejee ejfffd d Z Z S)JambaSdpaAttentionz Jamba attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from `JambaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. NFrYr%rrrrrr&cs|rtdtj||||||dS|\}} } ||} ||} ||} | || |j |j  dd} | || |j |j  dd} | || |j |j  dd} |dura| | | |j\} } t| |j} t| |j} |}|dur|ddddddd| jdf}| jjdkr|dur| } | } | } |jr|dur| dkrdnd}tjjj| | | ||jr|jnd |d }| dd}||| |j}||}|d|fS) NaJambaModel is using JambaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rYr%rrrrrr"rcudaTFr)Z attn_maskZ dropout_pr)rrrKr[rrrrrrrfrrdrrrgrr;r2typerrr3rr5Zscaled_dot_product_attentionrrrRr)rQrYr%rrrrrrrrBrrrrrrrTr'r+r[sV      &   zJambaSdpaAttention.forwardr) r^r_r`rr3rrrrhrrr[rar'r'rTr+rs2  r)eagerflash_attention_2sdpacseZdZdZdeffdd Z  ddejdede ej fd d Z ddede ej fd d Z  ddede ej fd dZ ZS)JambaMambaMixeru Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) rcst||_||_|j|_|j|_|j|_|j |j|_ |j |_ |j |_|j|_tj|j |j |j|j|j |jdd|_|j|_t|j|_|j|_tj|j|j d|jd|_tj|j |j |jddd|_tj|j |j dd|_td|jddddf}| |j d!}t"t#||_$t"t%|j |_&tj|j |j|jd|_'t(|j |j)d|_*t(|j|j)d|_+t(|j|j)d|_,t-st./d dSdS) Nr)Z in_channelsZ out_channelsrZ kernel_sizegroupspaddingr"rFTr/rSapThe fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d. If you want to use the naive implementation, set `use_mamba_kernels=False` in the model config)0rKrLrrrRrvrrwrrurZ mamba_dt_ranktime_step_rankZmamba_conv_bias use_conv_biasZmamba_proj_biasZuse_biasrConv1dconv1d hidden_act activationr actZuse_mamba_kernelsuse_fast_kernelsrin_projx_projdt_projr3aranger<rrMlogA_logrNDout_projrI rms_norm_eps dt_layernorm b_layernorm c_layernormis_fast_path_availablerr)rQrrArTr'r+rL-sJ   zJambaMambaMixer.__init__NrY cache_paramsr%c Cs|j\}}}|duo*|jo*|dko*|j|jjd|j|jjdko(|kn}||dd}|jddd\}} |durH||d}|j j |j j d|j j d} |rtt |d|j|j| |j j|j}|d}n'|durtj||j|jddf} |j|j| t|| |j j|jd}|dur||d}||dd} tj| |j|j|jgdd\} }}|| } ||}||}|jjj }t!t"|jjj |jj_ Wdn1swY|| dd}t! ||jj_ Wdn 1swYt#|j$% }|dur(|%nd}|rVt&|j|j|d|d||dddf|dddf|j'| d|dd d}n.t(||||dd|dd|j'%| |ddd \}}|dur|dur|j|j||)|dd}|S) Nrrr"r-r/)r).rT)Z dt_softplus)Zdelta_softplusZreturn_last_state)*r;rtrxrryrrchunkr?rrOrrr Zsqueezerrrr5padrcopy_rrr3splitrrrrrrdataZno_gradZ zeros_likeexprr:rrrr)rQrYrr%rEseq_lenrBZuse_precomputed_statesprojected_statesgateZ conv_weightsrxssm_parameters time_stepBCZtime_proj_biasdiscrete_time_stepr scan_outputs ssm_statecontextualized_statesr'r'r+cuda_kernels_forwardas  $                z$JambaMambaMixer.cuda_kernels_forwardc Cs|j\}}}|j}||dd}|jddd\} } |dur&| |d} t|t} | r|j|j jd|kr|j rD|j|j  } n|j|j } | | j } |jr|dkr|j|j jd|kr|j|j } tj| ddd} | dddddf| dddddf<| |j|j <tj| |jjdddddfdd} |jr| |jj7} ||  |d} n@tj| |j| jddf} | |j|j <||| dd|f} ntj||j|jf| j |d} ||| dd|f} |dur| |d} || dd}tj ||j!|j|jgdd\}}}|"|}|#|}|$|}|%|}tj&|dd}t'|j() }t'|ddddddf|dddddddf}|dddddddf|dddddddf)}|| dddddddf)}g}t*|D]D}|dddd|ddf| |dddd|ddf} t+| ||dd|ddfd}|,|dddddfqtj-|dd}|| |j.ddddf}||| }| r| |j|j <|/|dd}|S) Nrr"r-rr/)Zshiftsdims.rj)0r;rVrrrr?r0rhryrrcloner(r2rtrxr3Zrollr>rrOrrrrr5rrr|rrrrrrrrrZsoftplusrrr:r{rr}stackrr)rQZ input_statesrr%rErrBrVrrYrrrZ conv_staterrrrrrZ discrete_AZ discrete_BZdeltaB_urrZ scan_outputrr'r'r+ slow_forwardsx     ( (       :<$ <*  zJambaMambaMixer.slow_forwardcCs>|jrtr d|jjjjvrtd||||S||||S)NrzsFast Mamba kernels are not available. Make sure to they are installed and that the mamba module is on a CUDA device) rrrrOr2rrrr)rQrYrr%r'r'r+r[!szJambaMambaMixer.forwardr!)r^r_r`rrrLr3rrhrrrrr[rar'r'rTr+r%s(7 kXrcs$eZdZfddZddZZS)JambaMLPcsrt||_|j|_|j|_tj|j|jdd|_tj|j|jdd|_tj|j|jdd|_ t |j |_ dSNFr) rKrLrrRrrr gate_projup_proj down_projr ract_fnrQrrTr'r+rL2s zJambaMLP.__init__cCs$||||||}|Sr)r r r r )rQxr r'r'r+r[<s zJambaMLP.forward)r^r_r`rLr[rar'r'rTr+r1s  rcsFeZdZdZdeffdd ZdejdeejejffddZ Z S) JambaSparseMoeBlocka This implementation is strictly equivalent to standard MoE with full capacity (no dropped tokens). It's faster since it formulates MoE operations in terms of block-sparse operations to accommodate imbalanced assignments of tokens to experts, whereas standard MoE either (1) drop tokens at the cost of reduced performance or (2) set capacity factor to number of experts and thus waste computation and memory on padding. rcsdtj|_j|_j|_j|_t j |j|jdd|_ t fddt |jD|_dS)NFrcsg|]}tqSr')rrprr'r+r,Vsz0JambaSparseMoeBlock.__init__..)rKrLrR hidden_dimrZffn_dimr$num_experts_per_tokr@rrrouter ModuleListr{expertsrrTrr+rLNs $zJambaSparseMoeBlock.__init__rYr&cCs"|j\}}}|d|}||}tj|dtjd}tj||jdd\}}| |j }tj |||f|j |j d}tj jj||jdddd} t|jD]8} |j| } t| | \} } | jddkreqM|d | fd|}| ||| | d f}|d| | |j qM||||}||fS)  r/rrr-)rVr2)Z num_classesr"rN)r;rrFr6r3r:r7r@r(rVr|r2rr5r8r$Zpermuter{rwherer=Z index_add_)rQrYrErFrr#rArCZfinal_hidden_statesrDZ expert_idxZ expert_layeridxZtop_xZ current_stateZcurrent_hidden_statesr'r'r+r[Xs(     zJambaSparseMoeBlock.forward) r^r_r`rrrLr3rrr[rar'r'rTr+rBs ( rceZdZdedeffdd Z       ddejdeejd eej d ee d ee d ee d ee deej de ej ee ej ej fffddZZS)JambaAttentionDecoderLayerrrcsht|j|}t|j|||_|dkrtnt}|||_t |j |j d|_ t |j |j d|_ dS)Nrr)rKrLlayers_num_expertsJAMBA_ATTENTION_CLASSES_attn_implementation self_attnrr feed_forwardrIrRrinput_layernormpre_ff_layernormrQrrr$Zffn_layer_classrTr'r+rLs   z#JambaAttentionDecoderLayer.__init__NFrYr%rrroutput_router_logitsrrr&c  Cs|} ||}|j|||||||d\}} } | |}|} ||}||} t| tr0| \}} n| d}} | |}|f}|rC|| f7}|rJ|| f7}|rQ|| f7}|S) Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_value (`HybridMambaAttentionDynamicCache`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_router_logits (`bool`, *optional*): Whether or not to return the logits of all the routers. They are useful for computing the router loss, and should not be returned during inference. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. )rYr%rrrrrN)r"r r#r!r0r1)rQrYr%rrrr%rrresidualself_attn_weightsZpresent_key_value ff_outputsr#outputsr'r'r+r[s6           z"JambaAttentionDecoderLayer.forwardNNNFFFNr^r_r`rrrLr3rrrrhrrrr[rar'r'rTr+r8   rcr)JambaMambaDecoderLayerrrcsdt|j|}t||d|_|dkrtnt}|||_t|j |j d|_ t|j |j d|_ dS)N)rrrr) rKrLrrrirrr!rIrRrr"r#r$rTr'r+rLs   zJambaMambaDecoderLayer.__init__NFrYr%rrrr%rrr&c Cs|} ||}|j|||d}d} | |}|} ||}||} t| tr+| \}} n| d}} | |}|f} |r>| | f7} |rE| |f7} |rL| | f7} | S)r&)rYrr%N)r"rir#r!r0r1)rQrYr%rrrr%rrr'r(r)r#r*r'r'r+r[s0         zJambaMambaDecoderLayer.forwardr+r,r'r'rTr+r.r-r.c@s<eZdZeZdZdZddgZdZdZ dZ dZ dZ ddZ dS) JambaPreTrainedModelmodelTrr.rcCs |jj}t|tjtjfr%|jjjd|d|j dur#|j j dSdSt|tj rF|jjjd|d|j durD|jj|j  dSdSt|t rT|jjddSt|trtd|jddddf}||jd}|jjt||jjddSdS)Nr)r9stdg?rr/)rZinitializer_ranger0rrrrOrZnormal_rZzero_ Embedding padding_idxrIZfill_rr3rrr<rrrrrr)rQmoduler1rr'r'r+ _init_weights.s&     z"JambaPreTrainedModel._init_weightsN)r^r_r`rZ config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementZ_supports_flash_attn_2Z_supports_sdpaZ_supports_cache_classZ _is_statefulr5r'r'r'r+r/"s r/)Z attentionricseZdZdZdeffdd ZddZddZee dd e e j d e e j d e e j d e ede e jde ede ede ede ede e j defddZddZddZZS) JambaModelz Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`JambaDecoderLayer`] Args: config: JambaConfig rcst||j|_|j|_t|j|j|j|_g}t |j D]}t |j |}| |||dq t||_|j|_t|j|jd|_d|_|dS)N)rrF)rKrL pad_token_idr3 vocab_sizerr2rR embed_tokensr{rGALL_DECODER_LAYER_TYPESrsr}rlayersrrIrfinal_layernormgradient_checkpointing post_init)rQrZdecoder_layersrZ layer_classrTr'r+rLNs   zJambaModel.__init__cC|jSrr9r\r'r'r+get_input_embeddingsazJambaModel.get_input_embeddingscC ||_dSrr@rQvaluer'r'r+set_input_embeddingsd zJambaModel.set_input_embeddingsN input_idsr%rr inputs_embedsrroutput_hidden_statesr%rr&c  Cs|dur|n|jj}| dur| n|jj} |dur|n|jj}|dur$|n|jj}|du|duAr4td|jrC|jrC|rCt dd}|durL| |}|} |rY|durYt d| durht j | j d| jd} |durq| d}|||| } ||| } |rdnd}|rdnd}| rdnd}|jD]U}t|tr| n| }|r|| f7}|jr|jr||j| ||||| || }n || ||||| || d }|d} |r|ddur||df7}| r|d dur||d f7}q|| } |r|| f7}|r|jsd |_|sdn|}t| ||||d S) Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzJamba requires an initialized `HybridMambaAttentionDynamicCache` to return a cache. None was provided, so no cache will be returned.rrkrr')r%rrrr%rrr/T)last_hidden_staterrY attentionsr#)rrr%rJrrr=rrrr9r3rr;r2r?_update_causal_mask_update_mamba_maskr;r0r.Z_gradient_checkpointing_func__call__r<rtr)rQrHr%rrrIrrrJr%rrYr mamba_maskZall_hidden_statesZall_self_attnsZall_router_logitsZ decoder_layerZ layer_maskZ layer_outputsZ next_cacher'r'r+r[gs                 zJambaModel.forwardc Csv|jjdkr|durd|vr|SdS|j|j}}t|j}|jd}|dd}tj||f|||d} |dkr@tj | dd} | tj ||d| ddk9} | ddddddf |jdddd} |dur| } |d kr|jd} | d d| fd|ddddddfd} | d d| f| || d d| f<|jjd kr|dur|jjd vrt| |} | S) Nrrrr/)Z fill_valuerVr2)Zdiagonalrkrr".r)rZxpuZnpu)rrrVr2r3Zfinfominr;fullZtriurr=r<rr.eqZ masked_fillrr Z_unmask_unattended) rQr%Z input_tensorrrVr2Z min_dtyperFZ target_lengthrZ mask_lengthZ padding_maskr'r'r+rMs0    *  4$   zJambaModel._update_causal_maskcCs.|}|ddks|durt|dkrd}|S)zv No need for zeroing states when 1. Cached forward 2. Attending to all inputs rNr)r3all)rQr%rrPr'r'r+rNs"zJambaModel._update_mamba_mask) NNNNNNNNNN)r^r_r`rrrLrArFrrrr3rrrhrrrr[rMrNrar'r'rTr+r6EsT    s#r6cseZdZdgZdeffdd ZddZddZd d Zd d Z d dZ ddZ e e            d&deejdeejdeejdeedeejdeejdeedeedeedeedeejdeeejfdefd d!Z    "   #d'd$d%ZZS)(JambaForCausalLMzlm_head.weightrcsXt|t||_|j|_tj|j|jdd|_|j |_ |j |_ |j |_ | dSr ) rKrLr6r0r8rrrRlm_headrouter_aux_loss_coefr$rr>rrTr'r+rLs   zJambaForCausalLM.__init__cC|jjSrr0r9r\r'r'r+rArz%JambaForCausalLM.get_input_embeddingscC ||j_dSrrYrDr'r'r+rF z%JambaForCausalLM.set_input_embeddingscCr?rrVr\r'r'r+get_output_embeddings rBz&JambaForCausalLM.get_output_embeddingscCrCrr\)rQZnew_embeddingsr'r'r+set_output_embeddings#rGz&JambaForCausalLM.set_output_embeddingscCrCrr0)rQdecoderr'r'r+ set_decoder&rGzJambaForCausalLM.set_decodercCr?rr_r\r'r'r+ get_decoder)rBzJambaForCausalLM.get_decoderNrrHr%rrrIlabelsrrrJr%rlogits_to_keepr&c  Ks|dur|n|jj}| dur| n|jj} | dur| n|jj} |j|||||||| | | d }|j}t| tr;t| dn| }| |dd|ddf}d}|dur]|j |||j fi| }d}| rzt |j |j|j|}|durz||j||j7}t||||j|j|j|j dS)aj labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, JambaForCausalLM >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1") >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rHr%rrrIrrrJr%r)lossaux_losslogitsrrYrLr#)rrr%rJr0rKr0rslicerV loss_functionr8rHr#r$rrWr(r2rrrYrL)rQrHr%rrrIrcrrrJr%rrdZ loss_kwargsr*rYZ slice_indicesrgrerfr'r'r+r[,sT) zJambaForCausalLM.forwardFTc Ks|du} | s5|dus|d|jdkr"|dd|jd df}n!|jd|jdkr4|dd|f}nt|j|jd|j|jd}|durl|durl|dd}||dkd| sl|dd|jd df}|durw| rwd|i} nd|i} | ||||||jj |d| S)Nr/rrrkrIrH)rrrr%r%rdr) r;rhrrVr2longZcumsumZ masked_fill_rrZnum_logits_to_keep) rQrHrr%rIr%rrrrZ empty_past_kvZ model_inputsr'r'r+prepare_inputs_for_generations<    z.JambaForCausalLM.prepare_inputs_for_generation) NNNNNNNNNNNr)NNNFNNT)r^r_r`Z_tied_weights_keysrrLrArFr]r^rarbrrrr3rrrhrrrrrr[rkrar'r'rTr+rU st      ^rUa The Jamba Model with a sequence classification head on top (linear layer). [`JambaForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). )Z custom_introcseZdZfddZddZddZee         ddee j d ee j d ee j d ee d ee j d ee j deedeedeedefddZZS)JambaForSequenceClassificationcs@t||j|_t||_tj|j|jdd|_| dSr ) rKrLZ num_labelsr6r0rrrRscorer>rrTr'r+rLs   z'JambaForSequenceClassification.__init__cCrXrrYr\r'r'r+rArz3JambaForSequenceClassification.get_input_embeddingscCrZrrYrDr'r'r+rFr[z3JambaForSequenceClassification.set_input_embeddingsNrHr%rrrIrcrrrJr&c  Cs(|j|||||||| d} | j} || } |dur|jd} n|jd} |jjdur2| dkr2td|jjdur;d}n1|dur`||jjk| jt j }t j |jd| jt j d}|| d}n d}t |jjd| t j | | jd |f}d}|dur|j| |||jd }t||| j| j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). )r%rrrIrrrJNrrz=Cannot handle batch sizes > 1 if no padding token is defined.r/rjz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rk)rgrc pooled_logitsr)rergrrYrL)r0rKrmr;rr7rr(r2r3Zint32rZargmaxrrrUr^rirrrYrL)rQrHr%rrrIrcrrrJZtransformer_outputsrYrgrEZlast_non_pad_tokenZ non_pad_maskZ token_indicesrnrer'r'r+r[sL     z&JambaForSequenceClassification.forward) NNNNNNNNN)r^r_r`rLrArFrrrr3rrr rrrr[rar'r'rTr+rlsH     rl)rUrlr6r/)Nr"N)Mrrtypingrrrrrr3Ztorch.nn.functionalrr5rZtorch.utils.checkpointZ activationsr Z cache_utilsr r Z generationr Zmodeling_attn_mask_utilsr Zmodeling_flash_attention_utilsrrZmodeling_outputsrrrZmodeling_utilsrutilsrrrZutils.import_utilsrrZconfiguration_jambarrZ&mamba_ssm.ops.selective_scan_interfacerrZ+mamba_ssm.ops.triton.selective_state_updaterZ causal_conv1drr rTrZ get_loggerr^rrrrHModulerIrgrhrrrrrrrrr.r/r:r6rUrl__all__r'r'r'r+s            U X[`S?RO G; V