o Zh@sdZddlZddlmZddlmZmZmZmZm Z ddl Z ddl Z ddl m Z ddl mZddlmZdd lmZdd lmZdd lmZdd lmZmZmZdd lmZmZmZddlm Z e!e"Z#eroddl$m%Z%ndZ%erddl&m'Z'm(Z(ddl)m*Z*nd\Z*Z(Z'erddl+m,Z,m-Z-nd\Z-Z,e.e*e(e,e-e'fZ/Gddde j0Z1Gddde j0Z2Gddde j0Z3eGdddeZ4eGdddeZ5eGd d!d!eZ6eGd"d#d#e4Z7ed$d%Gd&d'd'e4eZ8gd(Z9dS))zPyTorch MAMBA model.N) dataclass)AnyDictOptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN) MambaCache)GenerationMixin)PreTrainedModel) ModelOutputauto_docstringlogging)is_causal_conv1d_availableis_mamba_ssm_availableis_mambapy_available) MambaConfig)pscan)mamba_inner_fnselective_scan_fn)selective_state_updateNNN)causal_conv1d_fncausal_conv1d_update)NNc seZdZdZdedeffdd Z   ddejde e d e ej d e ej fd d Z dde e d e ej d e ej fd dZ   dde e d e ej d e ej fddZZS) MambaMixeru Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) config layer_idxcst||_|j|_|j|_|j|_|j|_t |j |_ ||_ |j |_ t j|j|j|j |j|j|jdd|_|j|_t|j|_|j|_t j|j|jd|jd|_t j|j|j |jddd|_t j|j |jdd|_tjd|jdtjddddf}||jd}t t ||_!t t"|j|_#t j|j|j|jd|_$|j|_t%s|jrt&rt'(d dSt)d t'(d dSdS) Nr)Z in_channelsZ out_channelsbiasZ kernel_sizegroupspaddingr!FTdtypea7The fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the mamba.py backend. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzuse_mambapy is set to True but the mambapy package is not installed. To install it follow https://github.com/alxndrTL/mamba.py.aThe fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the sequential implementation of Mamba, as use_mambapy is set to False. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d. For the mamba.py backend, follow https://github.com/alxndrTL/mamba.py.)*super__init__r hidden_sizeZ state_sizessm_state_size conv_kernelconv_kernel_sizeintermediate_sizeinttime_step_rankr use_conv_biasrZConv1dconv1dZ hidden_act activationr act use_mambapyLinearuse_biasin_projx_projdt_projtorcharangefloat32expand contiguous ParameterlogA_logonesDout_projis_fast_path_availablerloggerZ warning_once ImportError)selfrr A __class__W/var/www/auris/lib/python3.10/site-packages/transformers/models/mamba/modeling_mamba.pyr*FsT    $zMambaMixer.__init__N hidden_states cache_paramscache_positionattention_maskcCs||dd}|jrI|durIt||jj|jr|jjnd|jj|j j|j j|j r/|j j ndt |j  dd|j |j j dd }|S|jddd\}}|dur]||d}|jj|jjd|jjd}|dur|ddkrt|d|j|j||jj|j}|d}n&|durtj||j|jddf} ||j| |t |||jj|jd}|dur||d}||dd} t j!| |j"|j#|j#gdd\} } } |j j| dd}t |j  }t$|j d r|j j nd}|dur2|ddkr2t%|j&|j|d |d || dddf| dddf|j|d |dd d}n,t'|||| dd| dd|j ||ddd \}}|dur^|dur^|(|j|| |dd}|S) Nrr$T)Z delta_biasdelta_softplusdimrr()r4r!).r)Z dt_softplus)rTZreturn_last_state))r9 transposetrainingrr3weightr2r!r:r;rFr8floatr<exprCrEchunk unsqueezeviewsizersqueeze conv_statesr r4r functionalpadr.shapeupdate_conv_statersplitr1r,hasattrr ssm_statesrZupdate_ssm_state)rJrPrQrRrSprojected_statescontextualized_statesgateZ conv_weightsrassm_parameters time_stepBCdiscrete_time_steprKZtime_proj_bias scan_outputs ssm_staterNrNrOcuda_kernels_forwards X$          zMambaMixer.cuda_kernels_forwardc Cs|j\}}}|j}||dd} | jddd\} } |dur&| |d} |dur|j|j} | | j } |jd|j krft j | |j | jddf} ||j| |||| dd|f} nU||j| |} | |jjj } tj| |jjdddddfdd} |jr| |jj7} ||  |d} ntj||j|jf| j |d} ||| dd|f} |dur| |d} || dd}tj||j|j|jgdd\}}}||}t j |dd}t|j ! }t|ddddddf|dddddddf}|dddddddf|dddddddf!}|| dddddddf!}|j"r|j#r|durt$|dd|dd}||d%ddd}|| |j&ddddf}||| }nug}t'|D]D}|dddd|ddf| |dddd|ddf} t(| ||dd|ddfd}|)|dddddfqtj*|dd}|| |j&ddddf}||| }|dur|j|j+| |,|dd}|S) Nrr$rUrr(.devicer'r )-rdr'r9rWr\r]rhr clonetorur.rrbrcrer5r3rYr<sumr2r!Zzerosr/r,r:rfr1r;Zsoftplusr[rCrZr6rXrr`rErangematmulappendstackcopy_rF)rJZ input_statesrQrRrSZ batch_sizeZseq_len_r'rirPrkrrZ conv_staterlrmrnrorprKZ discrete_AZ discrete_BZdeltaB_uhsZ scan_outputrqirjrNrNrO slow_forwardsp  (   :<$ <*  zMambaMixer.slow_forwardcCs>trd|jjjjvrtjs|||||S| ||||S)Ncuda) rGr:rYrutyper<Z_dynamoZ is_compilingrsr)rJrPrQrRrSrNrNrOforward:szMambaMixer.forwardr)__name__ __module__ __qualname____doc__rr0r*r<Tensorrr LongTensorrsrr __classcell__rNrNrLrOr>s4? (fUrcs.eZdZdfdd ZddZddZZS) MambaRMSNormư>cs&ttt||_||_dS)zL MambaRMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm N)r)r*rrAr<rDrYvariance_epsilon)rJr+epsrLrNrOr*Gs  zMambaRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)Nr$r(T)Zkeepdim) r'rwr<r>powmeanZrsqrtrrY)rJrPZ input_dtypeZvariancerNrNrOrOs  zMambaRMSNorm.forwardcCs|jjdd|jS)Nrz, eps=)rYrdrrJrNrNrO extra_reprVszMambaRMSNorm.extra_repr)r)rrrr*rrrrNrNrLrOrFsrcsJeZdZfddZ   d deedeejdeejfddZZ S) MambaBlockcsBt||_||_|j|_t|j|jd|_t ||d|_ dS)Nrr ) r)r*rr residual_in_fp32rr+layer_norm_epsilonnormrmixer)rJrr rLrNrOr*[s zMambaBlock.__init__NrQrRrScCsL|}||j|jjjd}|jr|tj}|j||||d}||}|S)Nr&rQrRrS)rrwrYr'rr<r>r)rJrPrQrRrSZresidualrNrNrOrcs zMambaBlock.forwardr) rrrr*rr r<rrrrNrNrLrOrZs  rc@s,eZdZeZdZddgZdZdZddZ dS)MambaPreTrainedModelbackbonerrTc Cst|trd|j_d|j_|jjd|jj}|jjdkr't j |j j |n|jjdkr8t j |j j | |tt|jjt|jjt|jjt|jjj|jjd}|tt|  }t|j j|Wdn1swYd|j j_t|t jr|jdurt|jddst j |jnt|t j!rt j j"|j |jj#d |jj$r|%D]2\}}|d vrt j j&|t'd d t|t'|jj(}Wdn1swYqdSdS) zInitialize the weights.TgZconstantrandom)minN _no_reinitF)Zstd)zout_proj.weight)a)) isinstancerrCZ_no_weight_decayrErr1Ztime_step_scaleZtime_step_init_schemerinitZ constant_r;rYZuniform_r<r[Zrandr/mathrBZ time_step_maxZ time_step_minclampZtime_step_floorexpm1Zno_gradr!r}rr7getattrZzeros_ EmbeddingZnormal_Zinitializer_rangeZrescale_prenorm_residualZnamed_parametersZkaiming_uniform_sqrtnum_hidden_layers)rJmoduleZ dt_init_stddtZinv_dtnameprNrNrO _init_weights~sN           z"MambaPreTrainedModel._init_weightsN) rrrrZ config_classZbase_model_prefixZ_no_split_modulesZsupports_gradient_checkpointingZ _is_statefulrrNrNrNrOrvs rc@sJeZdZUdZdZeejed<dZ ee ed<dZ ee ejed<dS) MambaOutputa# Class for the MAMBA model outputs. Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. cache_params (`MambaCache`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Includes both the State space model state matrices after the selective scan, and the Convolutional states hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. Nlast_hidden_staterQrP) rrrrrrr< FloatTensor__annotations__rQr rPrrNrNrNrOrs rc@s\eZdZUdZdZeejed<dZ eejed<dZ ee ed<dZ ee ejed<dS)MambaCausalLMOutputa Base class for causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). cache_params (`MambaCache`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Includes both the State space model state matrices after the selective scan, and the Convolutional states hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. NlosslogitsrQrP)rrrrrrr<rrrrQr rPrrNrNrNrOrs rcseZdZfddZddZddZddZe dd ee j d ee j d ee d ee dee dee dee j dee j de eeffddZZS) MambaModelcsnttjj|_tfddtj D|_ d|_ t jj d|_||j|dS)Ncsg|]}t|dqS)r)r).0idxrrNrO sz'MambaModel.__init__..Fr)r)r*rr vocab_sizer+ embeddingsZ ModuleListryrlayersgradient_checkpointingrrnorm_fZ"_register_load_state_dict_pre_hook load_hook post_initrJrrLrrOr*s    zMambaModel.__init__cGs2|D]}d|vr||||dd<dSqdS)Nz embedding.z embeddings.)popreplace)rJZ state_dictprefixargskrNrNrOrs zMambaModel.load_hookcC|jSNrrrNrNrOget_input_embeddingszMambaModel.get_input_embeddingscC ||_dSrrrJZnew_embeddingsrNrNrOset_input_embeddings zMambaModel.set_input_embeddingsN input_ids inputs_embedsrQ use_cacheoutput_hidden_states return_dictrRrSreturnc Cs|dur|n|jj}|dur|n|js|jjnd}|dur|n|jj}|du|duAr/td|dur8||}|jrB|jrB|rBd}|rk|durbt|j| d|j |j d}t j d|jj|j d}n |durjtdnd}|} |rsdnd} |jD]"} |jr|jr|| j| |||} n| | |||d } |r| | f} qx|| } |r| | f} |std d | || fDSt| |r|| d Sd| d S) a cache_params (`MambaCache`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. NFz:You must specify exactly one of input_ids or inputs_embedsrrtruzYou have to specify the `cache_position` manually when `use_cache=True` and `cache_params` is passed, you don't have to pass a `cache_params` if you are in prefilling stage because in that case it will be initialized for you automaticallyrNrcss|] }|dur|VqdSrrN)rvrNrNrO Isz%MambaModel.forward..)rrQrP)rrrXruse_return_dict ValueErrorrrr r_rur'r<r=r-rZ_gradient_checkpointing_func__call__rtupler) rJrrrQrrrrRrSrPZall_hidden_statesZ mixer_blockrNrNrOrsf         zMambaModel.forward)NNNNNNNN)rrrr*rrrrrr<rr boolrrrrrrNrNrLrOrsB    rz The MAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). )Z custom_introcs"eZdZdgZfddZddZddZdd Zd d Z d!d e de e e fde de e e ffddZ     d"deedeejdeejfddZe         d#deejdeejdeejdeedeejdeedeedeedeejdeeeffdd ZZS)$MambaForCausalLMzlm_head.weightcs8t|t||_tj|j|jdd|_| dS)NFr%) r)r*rrrr7r+rlm_headrrrLrNrOr*[s   zMambaForCausalLM.__init__cCrrrrrNrNrOget_output_embeddingsbrz&MambaForCausalLM.get_output_embeddingscCrrrrrNrNrOset_output_embeddingserz&MambaForCausalLM.set_output_embeddingscCs |jSr)rrrrNrNrOrhrz%MambaForCausalLM.get_input_embeddingscCs |j|Sr)rrrrNrNrOrks z%MambaForCausalLM.set_input_embeddingsroutputs model_kwargsnum_new_tokensrcKs|dd|d<|ddr$d|vr$|ddur$|ddd||d<d|vr?|d}tj|||jddfgdd |d<|S) NrQrTrRr(rSrrrU)getr<catZnew_onesrd)rJrrrkwargsrSrNrNrO#_update_model_kwargs_for_generationns   z4MambaForCausalLM._update_model_kwargs_for_generationNrQrRrSc Ks|r-|dur td|ddkr"|dddfd}|dur!d}n tjd|jj|jd}|dur:|dur:d|i}nd|i}|||||d|S)Nz`cache_position` should not be None as it should have been initialized in `model.generate`, you are responsible for passing in a valid `cache_position` if you are calling `prepare_inputs_for_generation` directly with `use_cache=True`rr(rrr)rQrrRrS) rr]r<r=rr-rur@update) rJrrrrQrRrSrZ model_inputsrNrNrOprepare_inputs_for_generations,    z.MambaForCausalLM.prepare_inputs_for_generationrrlabelsrrrc  Ks|dur|n|jj}|j||||||| |d} | d} || |jjj} d}|dur]|| j}| dddddf }|dddf }t }|| d| d| d}|ss| f| dd}|durq|f|S|St || | j| jdS)aS cache_params (`MambaCache`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. N)rQrrrrrRrSr.r(r)rrrQrP)rrrrrwrYr'rZrur@r r^r_rrQrP)rJrrSrrQrrrrrRrZ mamba_outputsrPrrZ shift_logitsZ shift_labelsZloss_fctoutputrNrNrOrs:  zMambaForCausalLM.forward)r)NNNNN) NNNNNNNNN)rrrZ_tied_weights_keysr*rrrrrrstrrr0rrr r<rrrrrrrrrrrrNrNrLrOrRsx     0   r)rrr):rr dataclassesrtypingrrrrrr<Ztorch.utils.checkpointrZtorch.nnr Z activationsr Z cache_utilsr Z generationr Zmodeling_utilsrutilsrrrZutils.import_utilsrrrZconfiguration_mambarZ get_loggerrrHZ mambapy.pscanrZ&mamba_ssm.ops.selective_scan_interfacerrZ+mamba_ssm.ops.triton.selective_state_updaterZ causal_conv1drrallrGModulerrrrrrrr__all__rNrNrNrOs`            7n