fTh 4SSKrSSKrSSKJrJrJr SSKrSSKrSSK J r SSK J s J r SSK Jr SSKJr SSKJr SSKJr SSKJrJrJrJrJrJr SS KJr SS KJrJ r J!r! S S K"J#r# \!RH"\%5r&"S S\ RN5r("SS\ RN5r)"SS\ RN5r*"SS\ RN5r+"SS\ RN5r,"SS\ RN5r-"SS\ RN5r."SS\ RN5r/"SS\ RN5r0"SS \ RN5r1\"S!S"\55r2"S#S$\ RN5r3"S%S&\ RN5r4"S'S(\ RN5r5"S)S*\ RN5r6"S+S,\ RN5r7"S-S.\ RN5r8SJS/\\9\94S0\:S1\9S2\\RvS3\9S4\Rx4 S5jjr=\r>\"S6S7\255r?S8r@\"S9S:9"S;S<\255rA\"S=S:9"S>S?\255rB\"S@SA\255rC"SBSC\ RN5rD"SDSE\ RN5rE\"SFS:9"SGSH\255rF/SIQrGg)KN)OptionalTupleUnion)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)auto_docstringis_peft_availablelogging) WavLMConfigc.^\rSrSrU4SjrSrSrU=r$)WavLMSamePadLayer$cR>[TU]5 US-S:XaSUlgSUlgNrr)super__init__num_pad_remove)selfnum_conv_pos_embeddings __class__s `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/wavlm/modeling_wavlm.pyrWavLMSamePadLayer.__init__%s) #:Q#>!#CacXURS:aUSS2SS2SUR*24nU$Nrrr hidden_statess r#forwardWavLMSamePadLayer.forward)s6    ")!Q0F43F3F2F0F*FGMr%r(__name__ __module__ __qualname____firstlineno__rr+__static_attributes__ __classcell__r"s@r#rr$sKr%rc.^\rSrSrU4SjrSrSrU=r$)WavLMPositionalConvEmbedding/c>[TU]5 [R"URURUR UR S-UR S9Ul[RRn[[RRS5(a$[RRRn[5(Ga%SSK nURRURR SS9 U"URSSS9UlSSS5 [URS5(aUURRR R"nURRR R$nO,URR&nURR(nURR+X5 URR+X5 OU"URSSS9Ul[-UR 5Ul[0UR2Ulg!,(df  GN,=f) Nr) kernel_sizepaddinggroups weight_normr) modifier_rankweight)namedimparametrizations)rrnnConv1d hidden_sizer!num_conv_pos_embedding_groupsconvutilsr<hasattrrAr deepspeedzeroGatheredParametersr> original0 original1weight_gweight_vregister_external_parameterrr:rfeat_extract_activation activation)r configr<rIrNrOr"s r#r%WavLMPositionalConvEmbedding.__init__0s II      6622a777   hh** 288,,m < <((33??K % ' ' 224993C3CST2U' aH Vtyy"4559955<<FF9955<<FF99--99-- NN 6 6t F NN 6 6t F#DIIH!DDI()G)GH  !?!?@VUs I IcURSS5nURU5nURU5nURU5nURSS5nU$Nrr) transposerFr:rRr)s r#r+$WavLMPositionalConvEmbedding.forwardQsV%//15  -0  ]3  6 %//15 r%)rRrFr:r-r4s@r#r6r6/sABr%r6c.^\rSrSrU4SjrSrSrU=r$)WavLMFeatureProjection\c4>[TU]5 [R"URSUR S9Ul[R"URSUR5Ul [R"UR5Ul g)Neps) rrrB LayerNormconv_dimlayer_norm_eps layer_normLinearrD projectionDropoutfeat_proj_dropoutdropoutr rSr"s r#rWavLMFeatureProjection.__init__]sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r%cnURU5nURU5nURU5nX4$N)rcrerh)r r*norm_hidden_statess r#r+WavLMFeatureProjection.forwardcs7!__];(:;  ]3 00r%)rhrcrer-r4s@r#rZrZ\s< 11r%rZc(^\rSrSrSrSS\S\S\S\S\S\4 U4S jjjrSS \ RS \ \ RS \ \ RS \S\ \ R\ \ R\ \ \ R44 Sjjr S \ RS \\ R \ R"4S\ RS \S\ R\ R44 SjrS\S\S\ R4SjrS\ RS\ R4SjrSrU=r$)WavLMAttentionkz=Multi-headed attention from 'Attention Is All You Need' paper embed_dim num_headsrh num_buckets max_distancehas_relative_position_biasc>[TU]5 XlX lX0lX-UlUR U-UR:wa[ SURSUS35eUR S-Ul[R"X5Ul [R"X5Ul [R"X5Ul [R"X5Ul X@lXPl[R "["R$"SURSS55Ul[R"UR S5UlU(a1[R*"URUR5Ulgg)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rrrrrsrhhead_dim ValueErrorscalingrBrdk_projv_projq_projout_projrtru Parametertorchonesgru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)r rrrsrhrtrurvr"s r#rWavLMAttention.__init__ns# "" !. MMI %$.. 8MdnnM]$YKr3 }}d* ii 5 ii 5 ii 5  )7 &(!#ejjDNNAq.Q!R"$))DMM1"= %"$,,t/?/?"PD  &r%r*attention_mask position_biasoutput_attentionsreturncUR5upgnUcPURXw5nURS5RUSSS5R X`R -Xw5nUR UR SSUR S4-5n U RSSSS5n URU 5n U R U R SSS-5RS5n [R"U 5RSSS9upXUR-S - -S -n U R X`R -SS5U-nUR SXw45nURXX5unnUUU4$) z'Attention layer with relative attentionNrrr]rr)rr@?g@)size compute_bias unsqueezerepeatviewrsshapepermutersumrsigmoidchunkrtorch_multi_head_self_attention)r r*rrrindexbsztgt_len_gated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightss r#r+WavLMAttention.forwards(,,.a   --g?M''*11#q!Q?DDS>>EY[bl  ,001D1DSb1IT^^]_L`1`a199!Q1E"&!8!89L!M!7!DDQBDO)?)? ?# EFL *..s^^/CRKm[166G7MN$($H$H +>% ! \L-77r%rcTURSS5=n=pgUbURS5OSnS=pSn [R"UUUURUR [ R"S/5[ R"URRURRURR45U U U URURRURRUR UUUSURRURRURRS9upU RSS5n U bCU SS2S4R#U R$SSUR 4-U R$SS-5n X4$)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)rWneFmulti_head_attention_forwardrrrsremptycatr~biasr|r}rhrr>training broadcast_tor)r r*rrrquerykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrs r#r.WavLMAttention.torch_multi_head_self_attentions,55a;;;3A3M>,,Q/SW %&$B$B    NN NN KK  IIt{{'')9)94;;;K;KL M    LL MM MM   MM   %)++,,++,,++,,+% ! 2"++Aq1  #(40==""2A&$..)::\=O=OPQPR=SSL((r% query_length key_lengthc~[R"U[RS9SS2S4n[R"U[RS9SSS24nXC- nURU5nUR UR R R5nUR U5nUR/SQ5nU$)Ndtype)rrr) rarangelong_relative_positions_buckettorr>devicer)r rrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluess r#rWavLMAttention.compute_biass << EJJG4P,,zDT1WM+>#'#B#BCT#U #;#>#>t?R?R?Y?Y?`?`#a $$%=> * r%relative_positionsc&URS-nUS:R[R5U-n[R"U5nUS-nX:n[R "UR 5U- 5nU[R "URU- 5- nXbU- -nXF-R[R5n[R"U[R"XrS- 55nU[R"XQU5- nU$r) rtrrrabslogfloatmathrumin full_likewhere)r rrtrelative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larges r#r)WavLMAttention._relative_positions_buckets&&!+ .266uzzB[P"YY'9:1$ %1&+ii0B0H0H0JY0V&W#&ADHHTM^M^ajMjDk&k#&AS\E\&]#&/&M%Q%QRWR\R\%]"%*YY &8RbcTc(d& " EKKF`aar%)rhrrrrryr|rurtrsrr~rr{r})i@i TNNFr)r.r/r0r1__doc__intrboolrrTensorrrr+ FloatTensorr LongTensor BoolTensorrrrr2r3r4s@r#rprpksG +/"Q"Q"Q "Q  "Q  "Q%)"Q"QN2604"' '8||'8!.'8  - '8  '8 u||Xell3XeELL>Q5RR S'8R5)((5)e..0@0@@A5)#.. 5)  5)   U.. / 5)n#%BSBS U=N=N SXSdSd  r%rpc.^\rSrSrU4SjrSrSrU=r$)WavLMFeedForwardic>[TU]5 [R"UR5Ul[R "URUR5Ul [UR[5(a[URUlOURUl[R "URUR5Ul[R"UR 5Ulgrl)rrrBrfactivation_dropoutintermediate_dropoutrdrDintermediate_sizeintermediate_dense isinstance hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutris r#rWavLMFeedForward.__init__s $&JJv/H/H$I!"$))F,>,>@X@X"Y f'' - -'-f.?.?'@D $'-'8'8D $IIf&>&>@R@RS jj)>)>?r%cURU5nURU5nURU5nURU5nUR U5nU$rl)rrrrrr)s r#r+WavLMFeedForward.forwardsX// > 00? 11-@ ))-8 ++M: r%)rrrrrr-r4s@r#rrs @r%rcB^\rSrSrSS\S\4U4SjjjrSSjrSrU=r $) WavLMEncoderLayeri)rSrvc >[TU]5 [URURUR UR URUS9Ul[R"UR5Ul [R"URURS9Ul[!U5Ul[R"URURS9UlgN)rrrsrhrtrurvr^rrrprDnum_attention_headsattention_dropoutrtmax_bucket_distance attentionrBrfrrhr`rbrcr feed_forwardfinal_layer_normr rSrvr"s r#rWavLMEncoderLayer.__init__* '((00,,**33'A  zz&"7"78 ,,v'9'9v?T?TU,V4 " V-?-?VEZEZ [r%cUnURUUUUUS9upnURU5nXa-nURU5nXRU5-nUR U5nX4nU(aX4- nU$)Nrrrr)rrhrcrr) r r*rrrr attn_residualroutputss r#r+WavLMEncoderLayer.forward9s% 59^^ )'/ 6D6 2 ] ]3 %5  6 %(9(9-(HH --m<  0   &Gr%rrhrrrcTr r.r/r0r1rrrr+r2r3r4s@r#rr)s) \{ \ \ \r%rcB^\rSrSrSS\S\4U4SjjjrSSjrSrU=r $) WavLMEncoderLayerStableLayerNormiRrSrvc >[TU]5 [URURUR UR URUS9Ul[R"UR5Ul [R"URURS9Ul[!U5Ul[R"URURS9Ulgrrrs r#r)WavLMEncoderLayerStableLayerNorm.__init__Srr%cUnURU5nURUUUUS9upnURU5nXQ-nXRUR U55-nX4nU(aXv4- nU$)N)rrr)rcrrhrr)r r*rrrrrrs r#r+(WavLMEncoderLayerStableLayerNorm.forwardbs%  6 59^^ )'/ 6D6 2 ]  ]3 %5 %(9(9$:O:OP]:^(__  0   &Gr%r r )NNFr r4s@r#r r Rs) \{ \ \ \r%r c:^\rSrSrU4SjrSSjrSrU=r$) WavLMEncoderiwc >[TU]5 Xl[U5Ul[ R "URURS9Ul [ R"UR5Ul [ R"[UR5Vs/sHn[!XS:HS9PM sn5UlSUlgs snfNr^r)rvF)rrrSr6pos_conv_embedrBr`rDrbrcrfrrh ModuleListrangenum_hidden_layersrlayersgradient_checkpointingr rSir"s r#rWavLMEncoder.__init__xs  :6B,,v'9'9v?T?TUzz&"7"78 mmUZ[a[s[sUt uUtPQ v6 KUt u  ',# v C c U(aSOSnU(aSOSnUb4URS5RSSURS5nSX)'URU5n X-nUR U5nUR U5n[ 5=(d [U5n Sn [UR5HupU(aXa4-n[R"/5nUR=(a$ U S:=(a XRR:nU(aU (aSUR(a1UR(a UR!U R"UUU U5nO U "UUU UU S9nUSSupU(aSnU(dMUWS4-nM U(aXa4-nU(d[%SXU455$['UUUS 9$) Nr]rrrrNNNc3.# UH ocMUv M g7frlr!.0vs r# 'WavLMEncoder.forward..m$[q$[ last_hidden_stater* attentions)rrrrrcrhr r enumeraterrrandrrS layerdropr_gradient_checkpointing_func__call__tupler r r*rroutput_hidden_states return_dictall_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusrrlayerdropout_probabilityskip_the_layer layer_outputss r#r+WavLMEncoder.forwards#7BD$5b4  %$2$<$[TU]5 Xl[U5Ul[ R "URURS9Ul [ R"UR5Ul [ R"[UR5Vs/sHn[!XS:HS9PM sn5UlSUlgs snfr)rrrSr6rrBr`rDrbrcrfrrhrrrr rrrs r#r$WavLMEncoderStableLayerNorm.__init__s  :6B,,v'9'9v?T?TUzz&"7"78 mmv778 8A1Z[U[]8   ',#  rcU(aSOSnU(aSOSnUb4URS5RSSURS5nSX)'URU5n X-nUR U5n[ 5=(d [ U5n Sn [UR5HupU(aXa4-n[R"/5nUR=(a$ U S:=(a XRR:nU(aU (aRUR(a1UR(a URU R UUU U5nO U "UUUU S9nUSSupU(aSnU(dMUWS4-nM UR#U5nU(aXa4-nU(d[%SXU455$['XUS 9$) Nr!r]rrr)rrrr"c3.# UH ocMUv M g7frlr!r$s r#r'6WavLMEncoderStableLayerNorm.forward..r)r*r+)rrrrrhr r r.rrr/rrSr0rr1r2rcr3r r4s r#r+#WavLMEncoderStableLayerNorm.forwards#7BD$5b4  %$2$<$[TU]5 URUlURUlUR UR-S:wa&[SUR SURS35e[R"[R"SURUR -UR UR-55Ul [R"URSURUR -5UlSUlg)Nrz`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.rr]r)rrnum_codevector_groups num_groupsnum_codevectors_per_groupnum_varscodevector_dimrzrBrrr codevectorsrdra weight_proj temperatureris r#r#WavLMGumbelVectorQuantizer.__init__$s  6688  4?? 2a 7)&*?*?)@A66:oo5FG%% <<   a4==!@&BWBW[_[j[jBj k 99V__R%8$//DMM:YZr%c URSS9n[R"[R"U[R"US-5-SS9*5R5nU$)NrrgHz>r])meanrexprr)probsmarginal_probs perplexitys r#_compute_perplexity.WavLMGumbelVectorQuantizer._compute_perplexity9sR*YY .599^VZEZ;[*[ac ddeiik r%cURup#nURU5nURX#-UR-S5nUR(a[ R RUR5URSS9nURU5n[R"URX#-URS5R5SS9nURU5nOyURSS9nUR"UR6R!SURSS5S5nURX#-URS5nURU5nURX#-S5nUR#S5UR$-n U RX#-URUR&S5n U R)S5RX#S5n X4$)Nr]T)tauhardrrr)rrTrrOrrB functionalgumbel_softmaxrrUtype_asrsoftmaxr]argmax new_zerosscatter_rrSrQr) r r* batch_sizesequence_lengthrDcodevector_probscodevector_soft_distr\codevector_idxcodevectors_per_grouprSs r#r+"WavLMGumbelVectorQuantizer.forward?s3@3F3F0 [((7 %**:+G$//+Y[]^ ==!}};;MAQAQ Q+001Mt`d`m`moqr !oob)..zBO &&r%)rSrOrQrUrT) r.r/r0r1rr staticmethodr]r+r2r3r4s@r#rLrLs+ * "'"'r%rLc\rSrSr\rSrSrSrSr Sr Sr SS\ \ R\4S \\4S jjrSS \S \ R4S jjrSrg)WavLMPreTrainedModelidwavlm input_valuesTFc [U[5(aURRRR SSS9 URR RR5 [RRUR5 g[U[5(a[RR URRSS[R"SURR SURR"-- 5-S9 [RR%URR S5 g[U[&5(a[R"SUR(R*- 5n[RRUR(RU*US9 [RRUR(R U*US9 g[U[R,5(akURRR SUR.R0S9 UR b%UR RR5 gg[U[R2[R445(aJUR RR5 URRR7S5 g[U[R85(a[RR;UR5 UR bh[R"UR<UR"UR S-- 5n[RRUR U*US9 ggg) zInitialize the weightsrr)rXstdrr)abNr)rrLrTr>datanormal_rzero_rBinituniform_rSr6rFrsqrtr9 in_channels constant_rZre in_featuresrdrSinitializer_ranger` GroupNormfill_rCkaiming_normal_r;)r moduleks r# _init_weights"WavLMPreTrainedModel._init_weightsms f8 9 9    % % * * 2 2 2 C    # # ( ( . . 0 GG  V// 0  < = = GGOO "" !v{{'>'>q'AFKKD[D['["\]]   GG  fkk.. 2  6 7 7 !f//;;;._conv_out_lengths 99\7wWZ[[ [r%r)rSrzip conv_kernel conv_stridernum_adapter_layersadapter_stride)r rrrr9rrs r# _get_feat_extract_output_lengths5WavLMPreTrainedModel._get_feat_extract_output_lengthss2=1Ddkk--+  \ $'t{{'>'> @W@W#X K,]PM$Y 4;;99: 04;;C]C] ^ ;r%feature_vector_lengthrcURSS9SS2S4nURXCS9nUR[R5nUR Sn[R "Xa4URURS9nSU[R"UR SURS9US- 4'URS/5RS5RS/5R5nU$)Nr]rrr)rrr)r) cumsumrrrrrzerosrrrflipr)r rrrnon_padded_lengthsoutput_lengthsrjs r#"_get_feature_vector_attention_mask7WavLMPreTrainedModel._get_feature_vector_attention_masks ,22r2:1b5A>>?Q>k'**5::6#))!,   /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr%r!rl)r.r/r0r1r config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_2_supports_sdparrrrrrrrrr2r!r%r#rsrsdsL$O&*#"N9DZ^"5#3#3S#89HPQU0Y]%(:?:J:Jr%rsc2^\rSrSrSU4SjjrSrSrU=r$)WavLMNoLayerNormConvLayericb>[TU]5 US:aURUS- OSUlURUUl[ R "URURURUURUURS9Ul [URUl g)Nrrr9rr)rrra in_conv_dim out_conv_dimrBrCrr conv_biasrFrrQrRr rSlayer_idr"s r#r"WavLMNoLayerNormConvLayer.__init__s [TU]5 US:aURUS- OSUlURUUl[ R "URURURUURUURS9Ul [ R"URSS9Ul [URUlg)NrrrT)elementwise_affine)rrrarrrBrCrrrrFr`rcrrQrRrs r#r WavLMLayerNormConvLayer.__init__s [TU]5 US:aURUS- OSUlURUUl[ R "URURURUURUURS9Ul [URUl [ R"URURSS9Ulg)NrrrT)rO num_channelsaffine)rrrarrrBrCrrrrFrrQrRrrcrs r#r WavLMGroupNormConvLayer.__init__s [TU]5 URS:Xa@[USS9/[ UR S- 5Vs/sHn[ XS-S9PM sn-nOVURS:Xa-[ UR 5Vs/sH n[XS9PM nnO[SURS35e[R"U5Ul SUl S Ul gs snfs snf) Ngroupr)rrr<z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)rrfeat_extract_normrrnum_feat_extract_layersrrrzrBr conv_layersr_requires_grad)r rSrrr"s r#rWavLMFeatureEncoder.__init__s   # #w .26AFGKPQWQoQorsQsKtKKta)&q5AKtKK % % 0PUV\VtVtPuvPu126FPuKvK01I1I0JJst ==5&+#"Kws C C%cNUR5H nSUlM SUlg)NF) parameters requires_gradrr params r#_freeze_parameters&WavLMFeatureEncoder._freeze_parameterss#__&E"'E '#r%cBUSS2S4nUR(aUR(aSUlURH\nUR(a@UR(a/UR(aUR UR U5nMTU"U5nM^ U$)NT)rrrrrr1r2)r rur* conv_layers r#r+WavLMFeatureEncoder.forwards$QW-    4==*.M '**J""t'B'Bt}} $ A A''!! !+= 9 +r%)rrr) r.r/r0r1rrrr+r2r3r4s@r#rrs8#"$ r%rc.^\rSrSrU4SjrSrSrU=r$)WavLMAdapterLayeri.c>[TU]5 [R"URSUR-UR UR SS9Ulg)Nrr)rr:)rrrBrCoutput_hidden_sizeadapter_kernel_sizerrFris r#rWavLMAdapterLayer.__init__/sJ II  % % )) )  & &((   r%cdURU5n[RRUSS9nU$)Nrr)rFrBrcglur)s r#r+WavLMAdapterLayer.forward9s/ -0  ))-Q)? r%)rFr-r4s@r#rr.s r%rc.^\rSrSrU4SjrSrSrU=r$) WavLMAdapteri@c>^[TU]5 TRTR:waV[R "TRTR5Ul[R"TR5UlOS=UlUl[R"U4Sj[TR555Ul TRUl g)Nc3:># UHn[T5v M g7frl)r)r%rrSs r#r'(WavLMAdapter.__init__..Ks#hGg!$5f$=$=Ggs)rrrrDrBrdprojr`proj_layer_normrrrrr0ris `r#rWavLMAdapter.__init__As   $ $(:(: : &"4"4f6O6OPDI#%<<0I0I#JD /3 3DI,mm#huVMfMfGg#hh ))r%c|URb/URb"URU5nURU5nURSS5nURHKn[R R 5nUR (aX0R:dMCU"U5nMM URSS5nU$rV)rrrWrnprandomrr0)r r*r<layerdrop_probs r#r+WavLMAdapter.forwardNs 99 T%9%9%E IIm4M 00?M%//15 [[EYY--/N==^nn%D %m 4 ! &//15 r%)r0rrrr-r4s@r#rr@s *r%rr mask_prob mask_lengthr min_masksrc ^^^^^UunmTS:a [S5eTT:a[STSTS35e[RRS5R 5mUUUUU4SjnUb-UR 5R S5R5O[U5Vs/sHnTPM snn[R"UT4[S 9n /n U"T5n U S :XaU $UHn U"U 5n [RR[R"U TS- - 5U S S 9n[U5S :XaTS- nOUS n[R"U[R"X- [R S 9U-/5nU R#U5 M [R$"U 5n [R&"U SS2SS2S4X[T45n U R)X[T-5n [R"T5SSSS24n[R&"UX[T45R)X[T-5nU U-n U R+5TS- :a TS- XTS- :'[R,"XSS5 U $s snf) a* Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `c>[TU-T- T-5n[UT5nUT-T:aTT-nUTS- - U:a[UTS- - S5nU$)z;Given input length, compute how many spans should be maskedrr)rmax)rnum_masked_spanepsilonrrrrks r#compute_num_masked_span6_compute_mask_indices..compute_num_masked_spansoi,6DwNOoy9 [ (? :-[TU]U5 Xl[U5Ul[ U5UlURS:dURS:aG[R"[R"UR5R55UlUR (a[#U5UlO['U5UlUR((a [+U5OSUlUR/5 g)Nr)rrrSrfeature_extractorrZfeature_projectionmask_time_probmask_feature_probrBrrrrDr~masked_spec_embeddo_stable_layer_normrDencoderrrradapter post_initris r#rWavLMModel.__init__s   !4V!<"8"@  3 &&*B*BS*H%'\\%,,v?Q?Q2R2[2[2]%^D "  & &6v>DL'/DL/5/A/A|F+t  r%cZ[R"S[5 UR5 gz Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nwarningswarn FutureWarningfreeze_feature_encoderr s r#freeze_feature_extractor#WavLMModel.freeze_feature_extractor'   Q  ##%r%c8URR5 g Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrr's r#r&!WavLMModel.freeze_feature_encoders 113r%r*mask_time_indicesrc[URSS5(dU$UR5upEnUb(URR UR 5X'OURR S:aUR(a[XE4URR URRUURRS9n[R"X!R[RS9nURR UR 5X'URRS:aUR(a[XF4URRURR URR"S9n[R"XqR[RS9nUSS2S4R%SUS5nSX'U$) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). apply_spec_augmentTNr)rrrr)rr)rrrr])getattrrSrrrrrrrmask_time_lengthmask_time_min_masksrtensorrrrmask_feature_lengthmask_feature_min_masksexpand)r r*r/rrjrkrDmask_feature_indicess r#_mask_hidden_statesWavLMModel._mask_hidden_statesst{{$8$?? 4A3E3E3G0 [  (/3/E/E/H/HI\I\/]M , [[ ' '! + 5-++44 KK88-++99 ! !& ->G[G[chcmcm n /3/E/E/H/HI\I\/]M , ;; ( (1 ,#8)++77 KK;;++<< $ $)<<0DMaMainisis#t #74#@#G#GO]_#` 23M /r%rurr5r6rc>UbUOURRnUbUOURRnUbUOURRnUR U5nUR SS5nUbUR URSUSS9nURU5upURXUS9nURUUUUUS9n U SnURbURU5nU(d X4U SS-$[UUU RU RS 9$) a mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. NrrFr)r/rrrr5r6r)r,extract_featuresr*r-)rSrr5use_return_dictrrWrrrr:rrWavLMBaseModelOutputr*r-) r rurr/rr5r6r>r*encoder_outputss r#r+WavLMModel.forward0sY2C1N-TXT_T_TqTq$8$D $++JjJj &1%uEN+/*A*ABR*S' 00 ~1 ,, )/!5# ' (* << # LL7M!4qr7JJ J#+-)77&11   r%)rrSrrrr)NNNNNNN)r.r/r0r1rrr(r&rrrrr:rrrrrr@r+r2r3r4s@r#rrs{( &4:>59 ,((,$E$5$56,!!1!12 ,\269=,0/3&*7 u||,7 !.7 $E$5$56 7 $D> 7 'tn 7 d^7  u** +7 7 r%rrzm WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). ) custom_introc^\rSrSrSS\\4U4SjjjrSrSrSr Sr \ SS\\ RS \\ RS \\S \\S \\S \\ RS\\\44Sjj5rSrU=r$) WavLMForCTCin target_langc>[TU]U5 [U5Ul[R "UR 5UlX lURc[SURS35e[US5(aUR(a URO URn[R "X1R5UlUR%5 g)a target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by default. NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.r)rrrrtrBrf final_dropoutrhrG vocab_sizerzr"rHrrrDrdlm_headr)r rSrGrr"s r#rWavLMForCTC.__init__ts  ' zz&"6"67 &    $00@AHH *1)G)GFL^L^F % %djdvdv yy!35F5FG  r%cURnUb'[URSS5c[SUS35eUc.[URSS5b[R S5 gUbUR USS9 gg)a This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. Nadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load)rGr2rSrzloggerinfo load_adapter)r rGs r# tie_weightsWavLMForCTC.tie_weightss&&  "wt{{ D 'tn D d^ D &D  un$ %D D r%rFz WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. c^\rSrSrU4SjrSrSrSr\SS\ \ RS\ \ RS\ \ S \ \ S \ \ S \ \ RS \ \\44S jj5rSrU=r$)WavLMForSequenceClassificationi c">[TU]U5 [US5(aUR(a [ S5e[ U5UlURS-nUR(a2[R"[R"U5U- 5Ul [R"URUR 5Ul[R"UR UR$5UlUR)5 g)Nrz\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrHrrzrrtruse_weighted_layer_sumrBrrr layer_weightsrdrDclassifier_proj_size projector num_labels classifierrr rS num_layersr"s r#r'WavLMForSequenceClassification.__init__s  6= ) )f.@.@n  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5v7R7RS))F$?$?ARARS r%cZ[R"S[5 UR5 gr r"r's r#r(7WavLMForSequenceClassification.freeze_feature_extractor!r*r%cLURRR5 gr,rYr's r#r&5WavLMForSequenceClassification.freeze_feature_encoder-r[r%cTURR5H nSUlM gr]r^rs r#r_0WavLMForSequenceClassification.freeze_base_model4rar%rurrr5r6rbrc0UbUOURRnURR(aSOUnURUUUUUS9nURR(aiU[n[ R "USS9n[RRURSS9n XRSSS5-RSS9nOUSnURU5nUcURSS9n OURUR SU5n U R#S5R%SSUR S5n S X)'URSS9U RSS9RSS5- n UR'U 5n SnUbF[)5nU"U RSURR*5URS55nU(dU 4U[S-nUbU4U-$U$[-UU UR.UR0S 9$) f input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTr=rrr]rrrrh)rSr?rrtrvrstackrBrcrfrrrrrXrrrrrrrr r*r-)r rurrr5r6rbrr* norm_weights pooled_output padding_maskexpand_padding_maskrjriloss_fctr{s r#r+&WavLMForSequenceClassification.forward<s,&1%->r1a-HHMMRSMTM#AJM}5  !)..1.5MBB=CVCVWXCY[ijL"."8"8"<"C"CAq-J]J]^_J`"a 25M. /)--!-4|7G7GA7G7N7S7STVXY7ZZM/  ')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$ Ev E'!//))   r%)rrrrtrC)r.r/r0r1rr(r&r_rrrrrrrr r+r2r3r4s@r#r~r~ s" &:(26,0/3&*)-A u||,A !.A $D> A 'tn A d^ A &A  u.. /A A r%r~c^\rSrSrU4SjrSrSrSr\SS\ \ RS\ \ RS\ \ RS \ \ S \ \ S \ \ S \ \\44S jj5rSrU=r$) WavLMForAudioFrameClassificationic>[TU]U5 [US5(aUR(a [ S5e[ U5UlURS-nUR(a2[R"[R"U5U- 5Ul [R"URUR 5UlUR UlUR%5 g)Nrz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrHrrzrrtrrrBrrrrrdrDrr init_weightsrs r#r)WavLMForAudioFrameClassification.__init__s  6= ) )f.@.@q  ' --1  ( (!#ejj.Dz.Q!RD ))F$6$68I8IJ ++ r%cZ[R"S[5 UR5 grVr"r's r#r(9WavLMForAudioFrameClassification.freeze_feature_extractorr*r%cLURRR5 gr,rYr's r#r&7WavLMForAudioFrameClassification.freeze_feature_encoderr[r%cTURR5H nSUlM gr]r^rs r#r_2WavLMForAudioFrameClassification.freeze_base_modelrar%rurrbrr5r6rc UbUOURRnURR(aSOUnURUUUUUS9nURR(aiU[n[ R "USS9n[RRURSS9n XRSSS5-RSS9nOUSnURU5n Sn UbZ[5n U "U RSUR5[ R "URSUR5SS95n U(dU 4U[S-n U $[#U U UR$UR&S 9$) rNTr=rrr]r)axisrh)rSr?rrtrvrrrBrcrfrrrrrrrgrr*r-)r rurrbrr5r6rr*rrjrirr{s r#r+(WavLMForAudioFrameClassification.forwardsf,&1%->r1a-HHMMRSMTM#AJM/  ')HFKKDOOtuDY)F)G!HHFM$!//))   r%)rrrrtrC)r.r/r0r1rr(r&r_rrrrrrrrr+r2r3r4s@r#rrs &:(26)-,0/3&*8 u||,8 !.8 & 8 $D> 8 'tn 8 d^8  u++ ,8 8 r%rc2^\rSrSrSU4SjjrSrSrU=r$) AMSoftmaxLossic>[[U] 5 X0lX@lX l[ R"[R"X5SS9Ul [ R"5Ul g)NT)r) rrrscalemarginrrBrrrandnr>rri)r input_dimrrrr"s r#rAMSoftmaxLoss.__init__sK mT+-  $ll5;;y#EUYZ '') r%cUR5n[RRURSS9n[RRUSS9n[ R "X5nX@R- n[RRX R5nUR[ R"UR5XT5-nURXr5nU$)Nrrr)flattenrBrc normalizer>rmmrone_hotrrrrri) r r*rbr> cos_thetapsionehotrjris r#r+AMSoftmaxLoss.forwards!((!(< // 1/E HH]3 ++%&&v?ekk&++-HHyy( r%)rirrrr>)g>@g?r-r4s@r#rrs*  r%rcf^\rSrSrSU4SjjrS\R S\R 4SjrSrU=r $) TDNNLayeric>[TU]5 US:aURUS- OURUUlURUUlUR UUlURUUl[R"URUR -UR5Ul [R"5Ul g)Nrr)rrtdnn_dimrr tdnn_kernelr9 tdnn_dilationdilationrBrdkernelReLUrRrs r#rTDNNLayer.__init__s [5(aSSKJn [5(a1[URW5(a[ R "S5 URSS5nURRRURURUR5RSS5n[RRXURR UR"S9nURSS5nUR%U5nU$)Nr) LoraLayerzDetected LoRA on TDNNLayer. LoRA weights won't be applied due to optimization. You should exclude TDNNLayer from LoRA's target modules.rr)r)rpeft.tuners.lorarrrr#r$rWr>rrr9rrBrcconv1drrrR)r r*rr>s r#r+TDNNLayer.forward s    2   $++y11 O &//15 ##(():):D[TU]U5 [U5UlURS-nUR (a2[ R"[R"U5U- 5Ul [ R"URURS5Ul[[!UR55Vs/sHn[#X5PM nn[ R$"U5Ul[ R"URSS-UR(5Ul[ R"UR(UR(5Ul[/UR(UR05UlUR55 gs snf)Nrrr]r)rrrrtrrrBrrrrrdrDrrrrrrtdnnxvector_output_dimrrrr objectiver)r rSrr tdnn_layersr"s r#rWavLMForXVector.__init__'s  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5vq7IJ5:3v;O5PQ5Py+5P QMM+. !#6??2+>+BFD]D]!^))F$=$=v?X?XY&v'@'@&BSBST RsFcZ[R"S[5 UR5 grVr"r's r#r((WavLMForXVector.freeze_feature_extractor:r*r%cLURRR5 gr,rYr's r#r&&WavLMForXVector.freeze_feature_encoderFr[r%cTURR5H nSUlM gr]r^rs r#r_!WavLMForXVector.freeze_base_modelMrar%rcXSnURRH nU"XS5nM U$)z/ Computes the output length of the TDNN layers cX- U-S-$)Nrr!rs r#rBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengthZs!.69A= =r%r)rSr)r rrr9s r#_get_tdnn_output_lengths(WavLMForXVector._get_tdnn_output_lengthsUs1  >  ;;22K,]KM3r%rurrr5r6rbrcUbUOURRnURR(aSOUnURUUUUUS9nURR(aiU[n[ R "USS9n[RRURSS9n XRSSS5-RSS9nOUSnURU5nURH n U "U5nM UcURSS9n UR!SS9n OUR#URSS95n UR%U 5n/n /n ['U5HNunnU R)XSU24RSS95 U R)XSU24R!SS95 MP [ R "U 5n [ R "U 5n [ R*"X/SS9nUR-U5nUR/U5nSnUbUR1UU5nU(dUU4U[S-nUbU4U-$U$[3UUUUR4UR6S9$) rNTr=rrr]r)rirj embeddingsr*r-)rSr?rrtrvrrrBrcrfrrrrrrXrwrrr.rrrrrrr*r-)r rurrr5r6rbrr*r tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsrjrir{s r#r+WavLMForXVector.forwardds,&1%->r1a-HHMMRSMTM#AJM}5 ))J&}5M$  !)..1.5M(,,,3L*.*O*OP^PbPbghPbPi*j '"&"?"?@["\ ML&':; 6$$]gvg:%>%C%C%C%JK##MWfW*$=$A$Aa$A$HI<"KK 6M ;;|4L!II}&CL 223DE!23  >>&&1D/07;X;Y3ZZF)-)9TGf$ Ev E(!//))   r%)rrrrrrrtrC)r.r/r0r1rr(r&r_rrrrrrrrrrrr+r2r3r4s@r#rr!s & &:( eE N 'tn N d^ N &N  um# $N N r%r)rrFr~rrrsr')Hrr#typingrrrnumpyrrtorch.nnrBtorch.nn.functionalrcrr activationsrintegrations.deepspeedr integrations.fsdpr modeling_outputsr r r rrrmodeling_utilsrrGrrrconfiguration_wavlmr get_loggerr.rPModulerr6rZrprrr rrDrLrsrrrrrrrrrndarrayrr@rrvrFr~rrrr__all__r!r%r#rs )) %!@7.??,   H % *299*Z 1RYY 1c RYYc Lryy0& &R"ryy"JP 299P fQ "))Q hC'C'LR?RRj *bii6bii0)")))X $299F26 t c?tttU--. t  t ZZ tn/N %N N b!" S &S  S l o %9o  o de ';e e PBII. @ M *M  M ` r%