o Zhr@sddlmZmZmZmZddlZddlZddlm Z ddl m Z ddl m Z mZmZddlmZddlmZmZmZddlmZdd lmZdd lmZmZmZmZmZdd l m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'ddl(m)Z)m*Z*m+Z+m,Z,ddl-m.Z.e+rddl/m0Z0ddl1m2Z2e,3e4Z5Gddde j6Z7Gddde j6Z8dej9de:dej9fddZ; dJde j6dej9dej9d ej9d!eej9d"edKd(d)Z?Gd*d+d+e j6Z@Gd,d-d-e j6ZAGd.d/d/eZBGd0d1d1eZCe)Gd2d3d3e%ZDGd4d5d5eDZEe)Gd6d7d7eDZF  dLd8ee:e:fd9ed?d?eDZJd@ej9dAe:dBe:fdCdDZKe)dEdFGdGdHdHeDeZLgdIZMdS)M)CallableOptionalTupleUnionN)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)AttentionMaskConverter_prepare_4d_attention_mask#_prepare_4d_attention_mask_for_sdpa)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPast)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringcan_return_tupleis_torch_flex_attn_availablelogging)MoonshineConfig) BlockMask)make_flex_block_causal_maskc2eZdZfddZdejdejfddZZS)MoonshineEncoderMLPcsBt||_t||_t|j|j|_ t|j|j|_ dSN super__init__configr activation_fnnnLinear hidden_sizeZintermediate_sizefc1fc2selfr)Z hidden_act __class___/var/www/auris/lib/python3.10/site-packages/transformers/models/moonshine/modeling_moonshine.pyr(=s  zMoonshineEncoderMLP.__init__ hidden_statesreturncCs"||}||}||}|Sr%)r.r*r/)r1r6r4r4r5forwardDs   zMoonshineEncoderMLP.forward__name__ __module__ __qualname__r(torchTensorr8 __classcell__r4r4r2r5r$< r$cr#)MoonshineDecoderMLPcsFt||_t||_t|j|jd|_ t|j|j|_ dS)Nr&r0r2r4r5r(Ls  zMoonshineDecoderMLP.__init__r6r7cCs8||}|jddd\}}|||}||}|S)NrBdim)r.chunkr*r/)r1r6Zgater4r4r5r8Ss  zMoonshineDecoderMLP.forwardr9r4r4r2r5rAKr@rAr6n_repr7cCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)shapeexpandreshape)r6rGbatchnum_key_value_headsslenhead_dimr4r4r5 repeat_kv[s 0rOmodulequerykeyvalueattention_maskscalingdropoutc Kst||j}t||j} t||dd|} |dur3|ddddddd|jdf} | | } tjj| dtj d |j } tjj | ||j d} t| | } | dd} | | fS)NrBrrC)rEdtype)ptrainingr)rOnum_key_value_groupsr=matmul transposerHr+ functionalZsoftmaxZfloat32torYrWr[ contiguous) rQrRrSrTrUrVrWkwargs key_states value_states attn_weights causal_mask attn_outputr4r4r5eager_attention_forwardgs & rhcCs>|ddddf}|ddddf}tj| |fdddS) z*Rotates half the hidden dims of the input..rNrBrrCrDrX)r=stackflatten)xx1Zx2r4r4r5 rotate_halfsrmc Cs||}||}|dd|jddfjddd}|dd|jddfjddd}|jd}|dd|f|d|df}}|dd|f|d|df} } ||t||} | |t| |} tj| |gdd} tj| | gdd} | | fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. .NrCrBrD) unsqueezerHZrepeat_interleavermr=cat) qkcossin position_idsZ unsqueeze_dimZ rotary_dimZq_rotZq_passZk_rotZk_passZq_embedZk_embedr4r4r5apply_rotary_pos_embs  $$ ""rucseZdZdZdededededef fdd Z dd ej d e e ej ej fd e ej d e e de ej de ej deede ej e ej e e ej ffddZZS)MoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr) layer_idx is_causalnum_attention_headsrLcst|||d||_||_t|d|j|j|_|j|j |_ |jd|_ |j |_ ||_ tj|j|j|j|jd|_tj|j|j |j|jd|_tj|j|j |j|jd|_tj|j|j|jdd|_|jjdur|jj}||j|d|}||j|_dSd|_dS)N)ryrLrNg࿩biasFrr)r'r(updater)rwgetattrr-ryrNrLr\rVattention_dropoutrxr+r,Zattention_biasq_projk_projv_projo_projZpad_head_dim_to_multiple_ofhead_dim_padding)r1r)rwrxryrLZtarget_multipleZtarget_head_dimr2r4r5r(s0    zMoonshineAttention.__init__Nr6position_embeddingsrUpast_key_valuecache_positionkey_value_statesrbr7cKs|jdd\}} |||| |jj|jdd} |du} |dur9|j|j } | r6d|j|j <|j }n|j }|dur?|n|} | rT|rT| rT|j |j }|j |j }n7|| |d|jj|jdd}|| |d|jj|jdd}| r|dur||||j d|i\}}| s|\}}t| |||\} }|dur|||d}||||j |\}}t}|jjdkr|jjdkr|d d rtd nt|jj}|jr|dur| dkrdnd }|jd krtjj| d |jf} tjj|d |jf}tjj|d |jf}||| |||f|jsd n|j|j|d|\}}|jd kr/|dd|j f}| || d!}|"|}||fS)NrCrrBTr)rsrrreagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rrP)rWrVrx.)#rHrviewr)rLrNr^ is_updatedgetrwcross_attention_cacheself_attention_cacheZ key_cacheZ value_cacherrr|rurh_attn_implementationlogger warning_oncerrxrr=r+r_padr[r~rVrJrar)r1r6rrUrrrrbZbszZq_lenZ query_statesZis_cross_attentionrZcurrent_statesrcrdrrrsZ cache_kwargsZattention_interfacerxrgrer4r4r5r8s "           zMoonshineAttention.forward)NNNNN)r:r;r<__doc__r intboolr(r=r>rrr LongTensorrrr8r?r4r4r2r5rvsF( rvcs8eZdZddeffdd ZeeddZZ S)MoonshineRotaryEmbeddingNr)cstt|dr|jdur|jd|jd|_nd|_|j|_|j|_||_ t |j|_ | |j |\}|_ |j d|dd|j|_dS)N rope_scaling rope_typetypedefaultinv_freqF) persistent)r'r(hasattrrrrZmax_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenr)rZ rope_init_fnattention_scalingZregister_bufferrZoriginal_inv_freq)r1r)devicerr2r4r5r(9s   z!MoonshineRotaryEmbedding.__init__c Cs|jddddf|jddd|j}|dddddf}t|jjtr6|jjdkr6|jjnd}t j |dd+|| dd}t j ||fdd }| |j}||j} Wdn1smwY|j|jd | j|jd fS) NrrCrZmpscpuF) device_typeenabledrBrDrY)rfloatrIrHr`r isinstancerstrr=Zautocastr^rorrrrsrY) r1rkrtZinv_freq_expandedZposition_ids_expandedrZfreqsZembrrrsr4r4r5r8Js0&z MoonshineRotaryEmbedding.forwardr%) r:r;r<r r(r=Zno_gradrr8r?r4r4r2r5r8s rcseZdZdedeffdd Z       ddejdeejd eej d ee d ee d ee d eej dee ejejfde ede ejee ejejfffddZZS)MoonshineEncoderLayerr)rwcs`t|j|_t||d|j|jd|_t||j|_ t j |jdd|_ t j |jdd|_ dS)NFr)rwrxryrLrz)r'r(r-rvZencoder_num_attention_headsZencoder_num_key_value_heads self_attnr$Zencoder_hidden_actmlpr+ LayerNorminput_layernormpost_attention_layernormr1r)rwr2r4r5r([s zMoonshineEncoderLayer.__init__NFr6rUrtrr use_cacherrrbr7c Kst|} ||}|jd||||||||d| \}} | |}|} ||}||}| |}|f} |r8| | f7} | S)Nr6rUrtrrrrrr4)rrrr) r1r6rUrtrrrrrrbresidualself_attn_weightsoutputsr4r4r5r8ks.      zMoonshineEncoderLayer.forward)NNNFFNN)r:r;r<r rr(r=r>rrrrrrr FloatTensorr8r?r4r4r2r5rZs<   rc seZdZddedeeffdd Z           ddejdeejd eejd eejd eej d eej d ee dee dee deej dee ejejfdee ejejfde ej ee ej ej fffddZZS)MoonshineDecoderLayerNr)rwcst|j|_t||d|j|jd|_t||d|j|jd|_t||j |_ t j |jdd|_ t j |jdd|_t j |jdd|_dS)NTrFrz)r'r(r-rvZdecoder_num_attention_headsZdecoder_num_key_value_headsr encoder_attnrAZdecoder_hidden_actrr+rrrfinal_layernormrr2r4r5r(s( zMoonshineDecoderLayer.__init__Fr6rUencoder_hidden_statesencoder_attention_maskrtencoder_position_idsrrrrrencoder_position_embeddingsr7c  Ks|}||}|jd|||||| | | d| \}}||}d}|dur<|}||}|j|||||| d\}}||}|}||}||}||}|f}|rW|||f7}|S)Nr)r6rrUrrrr4)rrrrrr)r1r6rUrrrtrrrrrrrrbrrZcross_attn_weightsrr4r4r5r8sH        zMoonshineDecoderLayer.forwardr%) NNNNNNFFNNN)r:r;r<r rrr(r=r>rrrrrr8r?r4r4r2r5rsP     rc@sLeZdZeZdZdZdZddgZdZ dZ dZ dZ ddZ dejfd d Zd S) MoonshinePreTrainedModelmodel input_valuesTrrcCs|jj}t|tjtjfr%|jjjd|d|j dur#|j j dSdSt|tj tj frD|jj d|j durB|j j dSdSt|tjrc|jjjd|d|jdure|jj|j dSdSdS)NrP)meanstdg?)r)Zinitializer_rangerr+r,Conv1dweightdataZnormal_r{Zzero_ GroupNormrZfill_ Embedding padding_idx)r1rQrr4r4r5 _init_weightss"    z&MoonshinePreTrainedModel._init_weights input_lengthscCs@t|ddd}t|ddd}t|ddd}|S)zH Computes the output length of the convolutional layers @rrrB)r)r1rZoutput_conv1_lengthZoutput_conv2_lengthZoutput_conv3_lengthr4r4r5 _get_feat_extract_output_lengthssz9MoonshinePreTrainedModel._get_feat_extract_output_lengthsN)r:r;r<r Z config_classZbase_model_prefixmain_input_nameZsupports_gradient_checkpointingZ_no_split_modulesZ_supports_flash_attn_2Z_supports_sdpaZ_supports_cache_classZ_supports_static_cacherr=rrr4r4r4r5rsrcseZdZdZdZdeffdd ZdejfddZ d ejfd d Z e dde e jd e e jde ede edeedef ddZZS)MoonshineEncoderz Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`] Args: config: MoonshineConfig rr)cst|_j}tjd|dddd|_tj|d|ddd |_tjd||ddd |_tj d|d d |_ t d |_ t fd dtjD|_tj|dd|_d|_|dS)NrrrF) kernel_sizestrider{rBrr)rrgh㈵>)Z num_groupsZ num_channelsZepsr)cg|]}t|qSr4)r.0idxrr4r5 *z-MoonshineEncoder.__init__..rz)r'r(r)r-r+rconv1conv2conv3r groupnormr rotary_emb ModuleListrangeZencoder_num_hidden_layerslayersr layer_normgradient_checkpointing post_init)r1r)Z embed_dimr2rr5r(s   zMoonshineEncoder.__init__r7cC|jSr%rr1r4r4r5get_input_embeddings1z%MoonshineEncoder.get_input_embeddingsrTcC ||_dSr%rr1rTr4r4r5set_input_embeddings4 z%MoonshineEncoder.set_input_embeddingsNrUroutput_hidden_statesflash_attn_kwargscKs|dur|n|jj}|dur|n|jj}|durtd|d}tj||}| |}tj | |}tj | |}| ddd}|dur||jd}d}|ddd|fdd|f}|jjd krv|d krs|nd}n|jjd kr|st||j}nt||j}tjd|jd|jd d} ||| } |rd nd} |rd nd} |jD]#} |r| |f7} | |f|| || d|}|d}|r| |df7} q||}|r| |f7} t|| | dS)a Args: input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): Float values of the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoFeatureExtractor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NzYou must specify input_values.rrrBrC.flash_attention_2rPrrr4)rUrtrrlast_hidden_stater6 attentions)r)rr ValueErrorrnr+r_tanhrrZgelurrZpermuterrHranyrrYr r=arangerrrrr)r1rrUrrrr6mask_lendownsample_stridertrall_hidden_statesall_self_attnsZ encoder_layer layer_outputsr4r4r5r87sb            zMoonshineEncoder.forward)NNNN)r:r;r<rrr r(r+Modulerrrrr=rr>rrrrr8r?r4r4r2r5rs0rcs6eZdZdZdeffdd ZddZddZee d#de e j d e e j d e e j d e ed e e jde ede ede ede e j de e jde e j deedeeeffddZ d$d ee j dfde j de j d edef ddZed e j dedede jde j d ef d!d"ZZS)%MoonshineDecoder input_idsr)cstj|_j|_tjj|j|_t fddt j D|_ tj jdd|_td|_d|_|dS)Ncrr4)rrrr4r5rrz-MoonshineDecoder.__init__..Frzr)r'r( pad_token_idr vocab_sizer+rr- embed_tokensrrZdecoder_num_hidden_layersrrnormrrrrr1r)r2rr5r(s   zMoonshineDecoder.__init__cCrr%rrr4r4r5rrz%MoonshineDecoder.get_input_embeddingscCrr%rrr4r4r5rrz%MoonshineDecoder.set_input_embeddingsNrUrtpast_key_values inputs_embedsrrrrrrrr7c  Ksv|dur|n|jj}|dur|n|jj}|dur|n|jj}|du|duAr*td|jr9|jr9|r9tdd}|durB| |}|rS|durSt } t }t | |}| duro|dur_| nd}t j|||jd|jd} |durx| d}|||| ||}|}|||}|rdnd}|rdnd}|r| durdnd}| dur| jd }d }| d dd|fd d|f} |jjd kr| d kr| nd} n|jjdkr|st| |j|jd } n t| |j|jd } |jD]5}|r||f7}||f|| | ||||| |d | }|d}|r||df7}| dur||df7}q||}|r-||f7}t||r4|nd|||dS)a encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Frrrr4rXr.rrPr) rUrrrtrrrrrrB)rrr6rcross_attentions)r)rrrrrr[rrrr r get_seq_lengthr=rrHrrn_update_causal_maskrrrrrYr rrr)r1rrUrtrr rrrrrrrrrpast_seen_tokensrfr6rrrZall_cross_attentionsrrZ decoder_layerrr4r4r5r8s                 zMoonshineDecoder.forwardFr! input_tensorc Cs:|jjdkr|dur|dkr|SdS|jjdkr&t|tjr$t|}|S|dur.|nd}|dur7|jnd}|jjdkrO|sO|sOt j ||||j drOdS|j }|j d} |r^|} nt|tjri|j d n|| d} |j|| | |||j dd } |jjdkr|dur|jjd vr|st|j} t | | } | S) NrrPZflex_attentionrFr)r Zpast_key_values_lengthZ is_trainingrrC)sequence_length target_lengthrYr batch_size)cudaZxpuZnpu)r)rrrr=r>r"r Zis_compileabler Z_ignore_causal_mask_sdpar[rYrHZget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionrrfinfominZ_unmask_unattended) r1rUrrrrr Zusing_compilable_cacherYrrrf min_dtyper4r4r5r =sT           z$MoonshineDecoder._update_causal_maskrrrYrc KsD|dur|dkr|}|St|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9}|ddddddf |ddd}|dur| }|j d} |ddddddd| f|ddddddf |j} | dk} |ddddddd| f | ||ddddddd| f<|S) aM Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. N)Z fill_valuerYrr)ZdiagonalrrCr)rEr=rrfullrZtriurrJrIclonerHr`Z masked_fill) rUrrrYrrrbrfr mask_lengthZ padding_maskr4r4r5rs,  $ 6  zFMoonshineDecoder._prepare_4d_causal_attention_mask_with_cache_position) NNNNNNNNNNN)F)r:r;r<rr r(rrrrrr=rr>rrrrrrrrr8r  staticmethodrrYrr?r4r4r2r5rs         DrrH mask_probr min_maskscs|\}dkr tdkrtdddtjdfdd}|dur:|d n fd d t|D}tj |ft d }g} |} | d krZ|S|D];} || } tjj t | d| dd} t | d kr}d}n| d }t| tj| | tjd |g} | | q\t| } t| dddddf|| f} | || } t ddddf}t||| f|| }| |} | dkrd| | dk<t|| dd |S)af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `csXt|}t|}|kr}|d|kr*t|dd}|S)z;Given input length, compute how many spans should be maskedrr)rmax) input_lengthnum_masked_spanepsilonrrrrr4r5compute_num_masked_spans  z6_compute_mask_indices..compute_num_masked_spanNrCcsg|]}qSr4r4)r_)rr4r5rsz)_compute_mask_indices..rrF)replace)rnprandomZranditemdetachsumtolistrZzerosrchoicerlenZ concatenateZonesZint32appendarrayZ broadcast_torJrZput_along_axis)rHrrrUrrr$rZ spec_aug_maskZspec_aug_mask_idxsZmax_num_masked_spanr r!Zspec_aug_mask_idxZdummy_mask_idxoffsetsr4r"r5_compute_mask_indicess\    r2cs&eZdZdeffdd ZddZddZdd Zd d Zd d Z d!de j de e j fddZee            d"de e j de e j de e j de e j de eee j de eeee j fde ee j de ee j de ede ede ede e j defdd ZZS)#MoonshineModelr)cs,t|t||_t||_|dSr%)r'r(rencoderrdecoderrrr2r4r5r(2s    zMoonshineModel.__init__cCs|jjSr%r5rrr4r4r5r:sz#MoonshineModel.get_input_embeddingscCs ||j_dSr%r6rr4r4r5r=s z#MoonshineModel.set_input_embeddingscCrr%)r4rr4r4r5 get_encoder@rzMoonshineModel.get_encodercCrr%)r5rr4r4r5 get_decoderCrzMoonshineModel.get_decodercCs|jdS)z Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will not be updated during training. N)r4Z_freeze_parametersrr4r4r5freeze_encoderFszMoonshineModel.freeze_encoderNinput_featuresrUcCst|jdds |S|\}}}|jjdkrE|jrEt||f|jj|jj||jjd}tj ||j tj d}|dddf d|d}d||<|jj dkrl|jrlt||f|jj |jj|jjd}tj ||j tj d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). Zapply_spec_augmentTr)rrrUr)rrYNrC)rrr)r}r)sizeZmask_time_probr[r2Zmask_time_lengthZmask_time_min_masksr=ZtensorrrrIZmask_feature_probZmask_feature_lengthZmask_feature_min_masks)r1r:rUrr-rZmask_time_indicesZmask_feature_indicesr4r4r5_mask_input_featuresMs0 z#MoonshineModel._mask_input_featuresrdecoder_input_idsdecoder_attention_maskencoder_outputsrdecoder_inputs_embedsdecoder_position_idsrrrrr7c  Cs| dur| n|jj} | dur| n|jj} | dur| n|jj} |dur,|j||| | d}n"t|tsNt|dt|dkr?|dndt|dkrJ|dndd}|j||||j |||| | | | d } t | j | j | j | j | j|j |j |j dS) a\ input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): Float values of the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoFeatureExtractor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) Example: ```python >>> import torch >>> from transformers import AutoFeatureExtractor, MoonshineModel >>> from datasets import load_dataset >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny") >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt") >>> input_values = inputs.input_values >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state >>> list(last_hidden_state.shape) [1, 2, 288] ``` N)rUrrrrrBr) rrUrrrr rtrrrr)rrdecoder_hidden_statesdecoder_attentionsr encoder_last_hidden_staterencoder_attentions)r)rrrr4rrr.r5rrrr6rr )r1rrUr=r>r?rr@rArrrrZdecoder_outputsr4r4r5r8xsPP zMoonshineModel.forwardr%) NNNNNNNNNNNN)r:r;r<r r(rrr7r8r9r=rrrr<rrrrr rrr8r?r4r4r2r5r30sl  +      r3rrdecoder_start_token_idcCsh||j}|ddddf|ddddf<||dddf<|dur*td||dk||S)z1 Shift input ids one token to the right. NrCrrz1self.model.config.pad_token_id has to be defined.i)Z new_zerosrHrrZ masked_fill_)rrrFZshifted_input_idsr4r4r5shift_tokens_rights (rGzj The Moonshine Model with a language modeling head. Can be used for automatic speech recognition. )Z custom_introcs"eZdZdgZdeffdd ZddZddZd d Zd d Z d e j fddZ e e             d deejdeejdeejdeejdeeeejdeeeeejfdeeejdeeejdeedeedeedeejdeejd efddZZS)!!MoonshineForConditionalGenerationzproj_out.weightr)cs8t|t||_tj|j|jdd|_| dS)NFrz) r'r(r3rr+r,r-rproj_outrrr2r4r5r(s   z*MoonshineForConditionalGeneration.__init__cC |jSr%)rr7rr4r4r5r7rz-MoonshineForConditionalGeneration.get_encodercCrJr%)rr8rr4r4r5r8rz-MoonshineForConditionalGeneration.get_decodercCrr%rIrr4r4r5get_output_embeddingsrz7MoonshineForConditionalGeneration.get_output_embeddingscCrr%rK)r1Znew_embeddingsr4r4r5set_output_embeddings!rz7MoonshineForConditionalGeneration.set_output_embeddingsr7cCrJr%)rrrr4r4r5r$rz6MoonshineForConditionalGeneration.get_input_embeddingsNrrUr=r>r?rr@rArrrrlabelscCs| dur|dur|durt| |jj|jj}|j||||||||| | | | d }||j}d}| dur=|j|| |jjd}t |||j |j |j |j |j|j|jd S)a input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): Float values of the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoFeatureExtractor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> import torch >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration >>> from datasets import load_dataset >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny") >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt") >>> input_values = inputs.input_values >>> generated_ids = model.generate(input_values, max_new_tokens=100) >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] >>> transcription 'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.' ```N) rUr=r?r>rr@rArrrr)logitsrNr) lossrOrrBrCr rDrrE)rGr)rrFrrIrZ loss_functionrrrrBrCr rDrrE)r1rrUr=r>r?rr@rArrrrrNrrOrPr4r4r5r8'sDY z)MoonshineForConditionalGeneration.forward) NNNNNNNNNNNNN)r:r;r<Z_tied_weights_keysr r(r7r8rLrMr+rrrrrr=rrrrr rrr8r?r4r4r2r5rHsh      rH)r3rrH)rP)Nr)Nr)Ntypingrrrrnumpyr'r=Ztorch.nnr+Z activationsrZ cache_utilsrr r Z generationr Zmodeling_attn_mask_utilsr r rZmodeling_flash_attention_utilsrZmodeling_layersrZmodeling_outputsrrrrrZmodeling_rope_utilsrrZmodeling_utilsrrZprocessing_utilsrutilsrrrrZconfiguration_moonshiner Z!torch.nn.attention.flex_attentionr!Zintegrations.flex_attentionr"Z get_loggerr:rrr$rAr>rrOrrhrmrurvrrrrrrrZndarrayr2r3rGrH__all__r4r4r4r5s            *";X%    wH