fThFSrSSKrSSKJrJrJr SSKrSSKrSSKJr SSK J r J r J r J r SSK Jr SSKJr SS KJr SS KJrJrJrJrJr SS KJr SS KJrJr S SKJ r \RB"\"5r#S&Sjr$"SS\RJ5r&"SS\RJ5r'"SS\RJ5r(\"SS\55r)\"SS\)55r*\"SS9"SS\)\55r+\"SS9"SS \)55r,\"S!S"\)55r-\"S#S$\)55r./S%Qr/g)'zPyTorch MPT model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLoss LayerNormMSELoss) functional)GenerationMixin)!_prepare_4d_causal_attention_mask))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)PreTrainedModel)auto_docstringlogging) MptConfigcH[R"SU- S[RUS9RSSSU5nS[R "[R "U55-n[R"SUS-[RUS9R5nXbU- -nS[R"SU5- nURSUSS5nXP:wa7[R"USS2SSS2S4USS2SSS2S4/SS9SS2SU2S4nXG-nURS5$) a Link to paper: https://arxiv.org/abs/2108.12409 - Alibi tensor is not causal as the original paper mentions, it relies on a translation invariance of softmax for quick implementation. This implementation has been copied from the alibi implementation of MPT source code that led to slightly different results than the Bloom alibi: https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L292 r)dtypedevice?N.dimr) torcharangeint32viewmathceillog2int64floatpowconcatsqueeze) num_headssequence_lengthalibi_bias_maxralibinum_heads_power_of_2baseslopess \/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/mpt/modeling_mpt.pybuild_mpt_alibi_tensorr4+s LL_,au{{6 R W WXY[\^_ap qE $))I*> ?? <</!35;;v V \ \ ^D $88 9D 599Q% %F [[0!Q 7F(vaAsl3VAssCK5HIqQRSU_V_U_adRde NE == c ^\rSrSrSrS\4U4SjjrS S\RS\RS\ \ \RS\ \R4S jjr S r U=r $) MptAttentionBzrMulti-head self attention. Using torch or triton attention implementation enables user to also use additive bias. configc>[TU]5 URUlURUlURUlURUR-UlURRUlURc5S[R"URUR- 5- UlURRUl URRUl [R"URSUR-SS9Ul[R"URURSS9Ulg)Nrr Fbias)super__init__ hidden_sizen_heads max_seq_lenmax_seq_lengthhead_dim attn_config softmax_scaler$sqrt attn_pdropattn_dropout_pclip_qkvrLinearWqkvout_projselfr9 __class__s r3r>MptAttention.__init__Gs !--~~ $00((DLL8 #//==    %!"TYYt/?/?$,,/N%O!OD $00;;**33 IId..D4D4D0D5Q  $"2"2D4D4D5Q r5 hidden_states position_biaspast_key_valueattention_maskcURSSupVURU5nUR(a%URUR*URS9nUR SSS9upn UR XVUR UR5RSS5nU R XVUR UR5RSS5n U R XVUR UR5RSS5n UbG[U5S:wa4[R"USU /SS9n [R"USU /SS9n X4nOX4n[R"XRSS55UR-n UcUOXcSRS-n Ub[UR5S:wa![S [UR535eU RSn [SUR!S5U - 5n[SUR!S5U - 5nUSS2US2US24nX-n Ub:U R#U[R$"UR&5R(5n [*R,R/U R15SS9R3U R&5n[*R,R5UUR6UR8S 9n[R"UU 5nUR;SSSS5R=5R?XVS5nURAU5nUUU4$) Nr)minmaxr rrrz6Expecting position_bias shape to be 3 dimensions, got ptraining)!shaperKrIclampchunkreshaper@rC transposelenr catmatmulrE ValueErrorrWsize masked_fillfinforrVrr softmaxr(todropoutrHr\permute contiguousr#rL)rNrQrRrSrT batch_size seq_length mixed_qkv query_states key_states value_statesattention_scores query_length key_lengthposition_bias_query_indexposition_bias_key_index attn_weightscontext_states attn_outputs r3forwardMptAttention.forwardVs"/!4!4Ra!8 IIm, ==!T]]N NI1:1J. ,#++JDLLRVR_R_`jjklnop ''  dmm\ffghjkl #++JDLLRVR_R_`jjklnop  %>"a'"YYq(9:'FAN $yy.*;\)JPQR (7N(7N << 6J6J2r6RSVZVhVhh%3%;z]^N_NeNefgNhAh  $=&&'1, #YZ]^k^q^qZrYs!tuu#))"-J(+A}/A/A!/D|/S(T %&)!]-?-?-BZ-O&P #)!-F-GI`Ia*abM/?   %/;;NEKKXdXjXjLkLoLop }},,-=-C-C-E2,NQQR^RdRde }},,\T=P=P[_[h[h,i lLA'//1a;FFHMMjfhimmN3 L.88r5) rKrHrIrCr?rBr@rLrE)NN)__name__ __module__ __qualname____firstlineno____doc__rr>r Tensorrrr|__static_attributes__ __classcell__rOs@r3r7r7Bsm Ry R&9=15 59||59||59!u||!45 59 !. 5959r5r7c^\rSrSrS\4U4SjjrS\RS\RS\R4SjrSr U=r $) MptMLPr9c>[TU]5 URn[R"USU-SS9Ul[R "SS9Ul[R"SU-USS9UlURRUl g)NFr;none) approximate) r=r>r?rrJup_projGELUact down_projrDrGhidden_dropoutrNr9r?rOs r3r>MptMLP.__init__sm (( yya+oEJ 77v.1{?KeL$00;;r5rQresidualreturncURURU55nURU5n[R"X0R UR S9nXB-nU$)NrZ)rrrFrkrr\)rNrQrintermediate_outputoutputs r3r|MptMLP.forwardsSm!<= "nn];.2E2EPTP]P]^" r5)rrrr) r~rrrrr>r rr|rrrs@r3rrs:[TU]5 URn[X!RS9UlSUR lURUl[U5Ul [X!RS9Ul SURl[U5Ul URRUl["R$"UR 5Ulg)Neps)r=r>r?r layer_norm_epsilonnorm_1r<r@r,r7attnnorm_2rffnrDrG dropout_raterDropoutresid_attn_dropoutrs r3r>MptBlock.__init__s ((  1J1JK   (  1J1JK  &>"..99"$**T->->"?r5rQrRrT layer_past use_cacheoutput_attentionscURU5nUnURUUUUS9upn URU 5U-nURU5nUnUR Xx5n U 4n U(aX4- n U(aX4- n U $)N)rRrTrS)rrrrr)rNrQrRrTrrrlayernorm_outputr attn_outputsryrSroutputss r3r|MptBlock.forwards ;;}5 6:YY ')% 6?6 2 N// =H ;;}5!*5)  ( (G   &Gr5)rrrrrr,r)NFF)r~rrrrr>r rrrboolr|rrrs@r3rrs@y@2CG"'(||(||( ( U5<<#=>? (  ( ((r5rc^\rSrSr\rSrSrS/rS/r U4Sjr S\ R4Sjr \S \\\R"\R"4S \\\R"\R"44S j5rS rU=r$) MptPreTrainedModel transformerTrz lm_head.*.c&>[TU]"U0UD6 gN)r=r>)rNinputskwargsrOs r3r>MptPreTrainedModel.__init__s &+F+r5modulec[U[R5(akURRR SUR RS9 URb%URRR5 gg[U[R5(axURRR SUR RS9 URb2URRURR5 gg[U[5(aWURb$URRR5 URRRS5 gg)zInitialize the weights.g)meanstdNr) isinstancerrJweightdatanormal_r9initializer_ranger<zero_ Embedding padding_idxr fill_)rNrs r3 _init_weights MptPreTrainedModel._init_weightss fbii ( ( MM   & &CT[[5R5R & S{{&   &&('  - - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  * *{{&   &&( MM   $ $S )+r5rSrcj^^^USSRupmmX-m[UUU4SjU55$)zg Converts the cache to the format expected by Mpt, i.e. to tuple(tuple([batch_size * num_heads, ...])) rc3|># UH1nUSRTTT5USRTTT54v M3 g7frrN)r`).0rbatch_size_times_num_headsrCros r3 ;MptPreTrainedModel._convert_to_mpt_cache.. sL - 1 %%&@(JW1 %%&@*hW -s9<)r]tuple)rSrnr,rrCros @@@r3_convert_to_mpt_cache(MptPreTrainedModel._convert_to_mpt_cachesI7EQ6G6J6P6P3 x%/%;" -    r5)r~rrrr config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_keys_to_ignore_on_load_missingr>rModuler staticmethodrr rrrrrs@r3rrsL%&*## '4o#,*BII*" eELL%,,$>?@ uU\\5<</0 1  r5rc~^\rSrSrS\4U4SjjrSrSSjrS\R4Sjr \ SS\ \RS \ \\\R\R4S 4S \ \RS \ \RS \ \S\ \S\ \S\ \S\\\RS 4\44Sjj5rSrU=r$)MptModelir9c>[TU]U5 URUlURUl[ R "URUR5Ul[ R"[UR5Vs/sHn[U5PM sn5Ul [URURS9UlSUR lSUlUR'5 gs snf)NrF)r=r>r?r@r,rr vocab_sizewte ModuleListrangen_layersrblocksr rnorm_fr<gradient_checkpointing post_init)rNr9_rOs r3r>MptModel.__init__s  !--<< 1 143C3CDmmuV__?U$V?U!Xf%5?U$VW   0 0f6O6OP  &+# %Ws C7cUR$rrrNs r3get_input_embeddingsMptModel.get_input_embeddings+s xxr5c[XX45$r)r4)rNr,r-r.rs r3r4MptModel.build_mpt_alibi_tensor.s%i.YYr5new_embeddingscXlgrrrNrs r3set_input_embeddingsMptModel.set_input_embeddings1s!r5 input_idspast_key_values.rT inputs_embedsrroutput_hidden_states return_dictrc UbUOURRnUbUOURRnUbUOURRnUbUOURRnUbUb [ S5eUbUR upOUbUR upn O [ S5eUc"[S/[UR5-5nUcURU5nUn U(aSOSnU(aSOSnU(aSOSnUR(a/UR(aU(a[RS5 SnU nSnUSbUSSR SnUU-nUc"[R "U U4U R"S 9nOUR%U R"5nUR'UR(URR*U R"S 9n[-X:U 4UU5nUR/5n[1URU5HunnU(aUU 4-nUR(a3UR(a"UR3UR4U UUUUU5nO U"U UUUUUS 9nUSn US La UUS 4-nU(dM~UUU(aSOS 4-nM UR7U 5n U(aUU 4-nU(d[S XUU455$[9U UUUS9$)h input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frrr)rrTrrrRTrc3.# UH ocMUv M g7frr)rvs r3r#MptModel.forward..sw$eq$es )last_hidden_staterrQ attentions)r9rrruse_return_dictrer]rrbrrrr\logger warning_oncer onesrrjr4r,rArrzip_gradient_checkpointing_func__call__rr)rNrrrTrrrrrrrnrorrQpresentsall_self_attentionsall_hidden_statesseq_length_with_pastpast_key_values_lengthr/ causal_maskblockrrs r3r|MptModel.forward4s42C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKcd d  "%.__ "J  &(5(;(; %JATU U  "#TFS-=$=>O   HHY/M% "2$5b4"6BD  & &4==##p"  *!" 1  )%4Q%7%:%@%@%C "#7:P#P  !"ZZ5I(JS`SgSghN+..}/C/CDN++DNNDKKOQd$eww w8+$+*   r5)rrr?rr,rNNNNNNNNN)r~rrrrr>rr4r rrrr LongTensorrrrrr|rrrs@r3rrs$y,Z"5<<"15SW1548$(,0/3&*{ E,,-{ "%ellELL.H(I3(N"OP{ !. {   0 01 { D> { $D>{ 'tn{ d^{  uU\\3&')RR S{ { r5rz The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introc8^\rSrSrS/rS\4U4SjjrSrS\R4Sjr \ SS\ \RS \ \\\R\R4S 4S \ \RS \ \RS \ \RS\ \S\ \S\ \S\ \S\\\R\44Sjj5rS\\\R\R4S 4S\RS\\\R\R4S 44SjrSrU=r$)MptForCausalLMizlm_head.weightr9c>[TU]U5 [U5Ul[R "UR URSS9UlUR5 gNFr;) r=r>rrrrJr?rlm_headrrMs r3r>MptForCausalLM.__init__sI  #F+yy!3!3V5F5FUS  r5cUR$rrrs r3get_output_embeddings$MptForCausalLM.get_output_embeddingss ||r5rcXlgrr rs r3set_output_embeddings$MptForCausalLM.set_output_embeddingss% r5rr.rTrlabelsrrrrrc U bU OURRn URUUUUUUUU S9n U Sn URU 5n SnUbEUR U R 5nUR "U U4SURR0U D6nU (dU 4U SS-nUbU4U-$U$[UU U RU RU RS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` NrrTrrrrrrrrlosslogitsrrQr) r9rrrrjr loss_functionrrrrQr)rNrrrTrr&rrrrrtransformer_outputsrQ lm_logitsr*rs r3r|MptForCausalLM.forwards>&1%# UHOnUSRSTUSR5USRSTUSR54v MQ g7fr) index_selectr)rrdevice_to_beam_idxs r3r0MptForCausalLM._reorder_cache..sf # 1 **1.@AAUAU.VW1 **1.@AAUAU.VW #sAA)rrjr)rNr0r1r past_statereordered_pastr5s @r3_reorder_cacheMptForCausalLM._reorder_cachesnQU PT*gqYc  x{{:+<+<= =gq PT  #    s8A)rr NNNNNNNNN)r~rrr_tied_weights_keysrr>r!r rr$rrrrrrrr|r9rrrs@r3rrs++y&ELL&15SW1504)-$(,0/3&*D E,,-D "%ellELL.H(I3(N"OPD !. D   - D & D D>D $D>D 'tnD d^D  uU\\"$EE FD D L% ell :;S@AMRM]M] uU\\5<</0#5 6r5ra The MPT Model transformer with a sequence classification head on top (linear layer). [`MptForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-1) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). ch^\rSrSrS\4U4Sjjr\SS\\RS\\ \ \R\R4S4S\\RS\\RS \\RS \\ S \\ S \\ S \\ S\ \ \R\44Sjj5rSrU=r$)MptForSequenceClassificationi)r9c>[TU]U5 URUl[U5Ul[ R "URURSS9UlUR5 gr) r=r> num_labelsrrrrJr?scorerrMs r3r>%MptForSequenceClassification.__init__8sV   ++#F+YYv1163D3D5Q  r5rr.rTrr&rrrrrc U bU OURRn URUUUUUUUU S9n U Sn URU 5n UbURSn OURSn URR cU S:wa [ S5eURR cSnOUbXRR :gRU R[R5n[R"URSU R[RS9nUU-RS5nO.Sn[RURR S35 U [R"XRS 9U4nSnUGbgURR"cUR$S:XaS URlOoUR$S:aNUR&[R(:XdUR&[R*:XaS URlOS URlURR"S :XaJ[-5nUR$S:Xa&U"UR/5UR/55nOeU"UU5nO[URR"S :Xa[15nU"UU5nO-URR"S :Xa[35nU"UU5nU (dU4U SS-nUbU4U-$U$[5UUU R6U R8U R:S 9$) input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr(rrz=Cannot handle batch sizes > 1 if no padding token is defined.rX)rrz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r regressionsingle_label_classificationmulti_label_classificationr))r9rrrAr] pad_token_idrerjrr r"r!argmaxrrrOr~ problem_typer@rlongintr r+rrrrrQr)rNrrrTrr&rrrrr-rQr+rnlast_non_pad_token non_pad_mask token_indices pooled_logitsr*loss_fctrs r3r|$MptForSequenceClassification.forwardAs<&1%!F!Fr!J !#    >>**+,ZZ  u||J}}MOaab   {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- v6))-II,. v6#%(;AB(??F)-)9TGf$ Ev E/ /??-;;*55   r5)r@rArr;)r~rrrrr>rrr rrrrrrr|rrrs@r3r>r>)sy15SW1504)-$(,0/3&*d E,,-d "%ellELL.H(I3(N"OPd !. d   - d & d D>d $D>d 'tnd d^d  uU\\"$DD Ed d r5r>ch^\rSrSrS\4U4Sjjr\SS\\RS\\ \ \R\R4S4S\\RS\\RS \\RS \\ S \\ S \\ S \\ S\ \ \R\44Sjj5rSrU=r$)MptForTokenClassificationir9c>[TU]U5 URUl[U5Ul[ US5(aUR b UR nO-[ US5(aURb URnOSn[R"U5Ul [R"URUR5Ul UR5 g)Nclassifier_dropoutrg?)r=r>r@rrhasattrrVrrrrkrJr? classifierr)rNr9rVrOs r3r>"MptForTokenClassification.__init__s   ++#F+ 6/ 0 0V5N5N5Z!'!:!:  V- . .63H3H3T!'!6!6 !$ zz"45 ))F$6$68I8IJ r5rr.rTrr&rrrrrc U bU OURRn URUUUUUUUU S9n U Sn URU 5n UR U 5n SnUblUR U R 5nURunn[5nU"U RUU-UR5URUU-55nU (dU 4U SS-nUbU4U-$U$[UU U RU RS9$)rDNr(rr)r*r+rQr)r9rrrkrXrjrr]rr#r@rrQr)rNrrrTrr&rrrrdeprecated_argumentsr-rQr+r*rnrorQrs r3r|!MptForTokenClassification.forwards+>&1%rrr rrrrrrr|rrrs@r3rTrTsy"15SW1504)-$(,0/3&*B E,,-B "%ellELL.H(I3(N"OPB !. B   - B & B D>B $D>B 'tnB d^B  uU\\"$99 :B B r5rTc^\rSrSrU4Sjr\SS\\RS\\RS\\RS\\RS\\RS\\ S \\ S \\ S \ \ \ 44S jj5rS rU=r$)MptForQuestionAnsweringic>[TU]U5 [U5Ul[R "UR S5UlUR5 g)Nr) r=r>rrrrJr? qa_outputsrrMs r3r> MptForQuestionAnswering.__init__sA  #F+))F$6$6: r5rrTrstart_positions end_positionsrrrrc UbUOURRnURUUUUUUS9n U Sn URU 5n U R SSS9upU R S5R 5n U R S5R 5n SnUbUb[UR55S:aUR S5n[UR55S:aUR S5nU RS5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- nU(dX4U SS-nUbU4U-$U$[UU U U RU RS 9$) rN)rTrrrrrrrXr) ignore_indexr)r* start_logits end_logitsrQr)r9rrr`splitr+rmrbrfr^rrrQr)rNrrTrrbrcrrrrsequence_outputr+rfrg total_loss ignored_indexrQ start_lossend_lossrs r3r|MptForQuestionAnswering.forward s2&1%rrr r FloatTensorrrrrr|rrrs@r3r^r^s156:596:48,0/3&*E E,,-E !!2!23E   1 12 E "%"2"23 E   0 01 E $D>E 'tnE d^E  u22 3E E r5r^)rrrr>rTr^r)0rr$typingrrrr torch.utils.checkpointrtorch.nnrrr r r r generationr modeling_attn_mask_utilsrmodeling_outputsrrrrrmodeling_utilsrutilsrrconfiguration_mptr get_loggerr~rr4rr7rrrrrr>rTr^__all__rr5r3r{sd )) LL$)I.,(   H %.I9299I9XRYY*=ryy=@, , , ^\ !\ \ ~ m'm  m`  o #5o  o dU 2U U pO 0O O d r5