o ZhH@sdZddlmZddlZddlmZddlmZmZm Z ddl Z ddl Z ddlmZddlmZmZmZmZmZmZmZmZdd lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0d d l1m2Z2e/3e4Z5dZ6dZ7dZ8dZ9dZ:dZ;dZdZ?dZ@dZAdZBGdddZCGddde#jDjEZFGdd d e#jDjGZHGd!d"d"e#jDjEZIeHeId#ZJGd$d%d%e#jDjEZKGd&d'd'e#jDjEZLGd(d)d)e#jDjEZMGd*d+d+e#jDjEZNGd,d-d-e#jDjEZOGd.d/d/e#jDjEZPGd0d1d1e#jDjEZQGd2d3d3e#jDjEZRGd4d5d5e#jDjEZSGd6d7d7e#jDjEZTGd8d9d9e#jDjEZUGd:d;d;e#jDjEZVGdd?d?e#jDjEZXGd@dAdAe#jDjEZYGdBdCdCe#jDjEZZe$GdDdEdEe#jDjEZ[GdFdGdGeZ\eGdHdIdIe+Z]dJZ^dKZ_e-dLe^GdMdNdNe\Z`e-dOe^GdPdQdQe\eCZae-dRe^GdSdTdTe\eZbGdUdVdVe#jDjEZce-dWe^GdXdYdYe\eZde-dZe^Gd[d\d\e\e Zee-d]e^Gd^d_d_e\eZfe-d`e^Gdadbdbe\eZge-dce^Gdddedee\e!ZhgdfZidS)gzTF 2.0 MobileBERT model.) annotationsN) dataclass)OptionalTupleUnion)get_tf_activation)TFBaseModelOutputTFBaseModelOutputWithPoolingTFMaskedLMOutputTFMultipleChoiceModelOutputTFNextSentencePredictorOutputTFQuestionAnsweringModelOutputTFSequenceClassifierOutputTFTokenClassifierOutput) TFMaskedLanguageModelingLossTFModelInputTypeTFMultipleChoiceLossTFNextSentencePredictionLossTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFTokenClassificationLossget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)MobileBertConfigzgoogle/mobilebert-uncasedr'z"vumichien/mobilebert-finetuned-nerzK['I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-LOC', 'I-LOC']gQ?z%vumichien/mobilebert-uncased-squad-v2z'a nice puppet'gףp= @ zvumichien/emo-mobilebertz'others'z4.72c@seZdZdZd ddZdS) TFMobileBertPreTrainingLossz Loss function suitable for BERT-like pretraining, that is, the task of pretraining a language model by combining NSP + MLM. .. note:: Any label of -100 will be ignored (along with the corresponding logits) in the loss computation. labels tf.Tensorlogitsreturnc Cstjjdtjjjd}|tj|d|dd}tj|ddk|j d}||}t |t |}|tj|d|d d}tj|ddk|j d} || } t | t | } t || d S) NT)Z from_logitsZ reductionr+r)Zy_trueZy_predidtypenext_sentence_labelr&)r&) rZlossesZSparseCategoricalCrossentropyZ ReductionNONEtfnnZrelucastr0Z reduce_sumreshape) selfr+r-Zloss_fnZunmasked_lm_lossesZ lm_loss_maskZmasked_lm_lossesZreduced_masked_lm_lossZunmasked_ns_lossZ ns_loss_maskZmasked_ns_lossZreduced_masked_ns_lossr8d/var/www/auris/lib/python3.10/site-packages/transformers/models/mobilebert/modeling_tf_mobilebert.pyhf_compute_loss_sz+TFMobileBertPreTrainingLoss.hf_compute_lossN)r+r,r-r,r.r,)__name__ __module__ __qualname____doc__r:r8r8r8r9r*Xsr*c.eZdZfddZddZdddZZS) TFMobileBertIntermediatec sRtjdi|tjj|jdd|_t|jt r t |j|_ n|j|_ ||_ dS)Ndensenamer8) super__init__rlayersDenseintermediate_sizerA isinstance hidden_actstrrintermediate_act_fnconfigr7rMkwargs __class__r8r9rEus   z!TFMobileBertIntermediate.__init__cCs||}||}|SN)rArLr7 hidden_statesr8r8r9calls  zTFMobileBertIntermediate.callNcCn|jrdSd|_t|dddur5t|jj|jdd|jjgWddS1s.wYdSdSNTrA) builtgetattrr3 name_scoperArCbuildrMtrue_hidden_sizer7 input_shaper8r8r9r["zTFMobileBertIntermediate.buildrRr;r<r=rErUr[ __classcell__r8r8rPr9r@ts  r@cs*eZdZfddZdfdd ZZS) TFLayerNormcs||_tj|i|dSrR) feat_sizerDrE)r7rcargsrOrPr8r9rEszTFLayerNorm.__init__Ncstdd|jgdSrR)rDr[rcr]rPr8r9r[szTFLayerNorm.buildrR)r;r<r=rEr[rar8r8rPr9rbs rbcs4eZdZd fdd ZfddZd dd ZZS) TFNoNormNc stjdi|||_dS)Nr8)rDrErc)r7rcepsilonrOrPr8r9rEs zTFNoNorm.__init__cs<|jd|jgdd|_|jd|jgdd|_t|dS)Nbiaszeros)shape initializerweightZones) add_weightrcrgrkrDr[r]rPr8r9r[szTFNoNorm.buildinputsr,cCs||j|jSrR)rkrg)r7rmr8r8r9rUsz TFNoNorm.callrR)rmr,)r;r<r=rEr[rUrar8r8rPr9res re) layer_normZno_normcs4eZdZdZfddZd ddZd dd ZZS) TFMobileBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c stjdi||j|_|j|_||_|j|_|j|_|j|_tj j |jdd|_ t |j |j|jdd|_tj j|jd|_|j|jrGdnd|_dS) Nembedding_transformationrB LayerNormrfrC)Zraterr&r8)rDrE trigram_inputembedding_sizerM hidden_sizemax_position_embeddingsinitializer_rangerrFrGrpNORM2FNnormalization_typelayer_norm_epsrqDropouthidden_dropout_probdropoutembedded_input_sizerNrPr8r9rEs zTFMobileBertEmbeddings.__init__NcCstd|jd|jj|jgt|jdd|_Wdn1s"wYtd|jd|jj |j gt|jdd|_ Wdn1sIwYtd|jd|j |j gt|jdd|_ Wdn1sowY|jrydSd|_t|d ddurt|jj|jdd|jgWdn1swYt|d ddurt|jj|jdWddS1swYdSdS) NZword_embeddingsrk)rw)rCrirjtoken_type_embeddings embeddingsposition_embeddingsTrprq)r3rZrlrM vocab_sizertrrwrkZtype_vocab_sizerurrvrrXrYrprCr[r~rqr]r8r8r9r[sB            "zTFMobileBertEmbeddings.buildFc Cs4|dur |dur J|durt||jjtj|j|d}t|dd}|dur0tj|dd}|jrUtj t |ddddfd|t |ddddfdgd d }|js^|j |j krc| |}|durutjtjd|dd dd }tj|j|d}tj|j|d}|||} |j| d } |j| |d } | S)z Applies embedding based on inputs tensor. Returns: final_embeddings (`tf.Tensor`): output embedding tensor. N)paramsindicesr)dimsvaluer&)rr)rr&r)r)r&rrZaxis)startlimit)rm)rmtraining)rrMrr3gatherrkrfillrsconcatpadrtrurpZ expand_dimsrangerrrqr}) r7 input_ids position_idstoken_type_ids inputs_embedsrr^Zposition_embedsZtoken_type_embedsZfinal_embeddingsr8r8r9rUs2   zTFMobileBertEmbeddings.callrR)NNNNF)r;r<r=r>rEr[rUrar8r8rPr9ros    roc:eZdZfddZddZ d ddZd d d ZZS) TFMobileBertSelfAttentionc stjdi||j|jdkrtd|jd|j|j|_|j|_|j|jdks/Jt|j|j|_|j|j|_ t j j |j t |jdd|_t j j |j t |jdd|_t j j |j t |jdd|_t j |j|_||_dS) NrzThe hidden size (z6) is not a multiple of the number of attention heads (querykernel_initializerrCkeyrr8)rDrErunum_attention_heads ValueErroroutput_attentionsintr\attention_head_size all_head_sizerrFrGrrwrrrr{Zattention_probs_dropout_probr}rMrNrPr8r9rEs.  z"TFMobileBertSelfAttention.__init__cCs*t||d|j|jf}tj|gddS)Nrrrr&rperm)r3r6rr transpose)r7x batch_sizer8r8r9transpose_for_scores-sz.TFMobileBertSelfAttention.transpose_for_scoresFcCst|d}||} ||} ||} || |} || |} || |}tj| | dd}tjt| d|jd}|tj |}|durStj||jd}||}t |dd}|j ||d}|durh||}t||}tj |gdd }t||d|jf}|r||f}|S|f}|S) NrT)Z transpose_brr/rrrr)rrrrrr3matmulr5r0mathsqrtrr}rr6r)r7 query_tensor key_tensor value_tensorattention_mask head_maskrrrZmixed_query_layerZmixed_key_layerZmixed_value_layerZ query_layerZ key_layerZ value_layerZattention_scoresZdkZattention_probsZ context_layeroutputsr8r8r9rU2s8           zTFMobileBertSelfAttention.callNcCs&|jrdSd|_t|dddur2t|jj|jdd|jjgWdn1s-wYt|dddur\t|j j|j dd|jjgWdn1sWwYt|dddurt|j j|j dd|jj ry|jjn|jj gWddS1swYdSdS)NTrrr) rXrYr3rZrrCr[rMr\rruse_bottleneck_attentionrur]r8r8r9r[_s. "zTFMobileBertSelfAttention.buildFrR)r;r<r=rErrUr[rar8r8rPr9rs   -rc0eZdZfddZd ddZd ddZZS) TFMobileBertSelfOutputc sptjdi||j|_tjj|jt|jdd|_ t |j |j|j dd|_ |js3tj|j|_||_dS)NrArrqrrr8)rDrEuse_bottleneckrrFrGr\rrwrArxryrzrqr{r|r}rMrNrPr8r9rEws  zTFMobileBertSelfOutput.__init__FcCs0||}|js|j||d}|||}|SNr)rArr}rq)r7rTresidual_tensorrr8r8r9rUs zTFMobileBertSelfOutput.callNcC|jrdSd|_t|dddur2t|jj|jdd|jjgWdn1s-wYt|dddurZt|j j|j dWddS1sSwYdSdSNTrArq rXrYr3rZrArCr[rMr\rqr]r8r8r9r["zTFMobileBertSelfOutput.buildrrRr`r8r8rPr9rvs  rcr) TFMobileBertAttentionc 2tjdi|t|dd|_t|dd|_dS)Nr7rBoutputr8)rDrErr7rmobilebert_outputrNrPr8r9rEzTFMobileBertAttention.__init__cCtrRNotImplementedError)r7Zheadsr8r8r9 prune_headsz!TFMobileBertAttention.prune_headsFc CsB|j|||||||d} |j| d||d} | f| dd} | S)Nrrr&)r7r) r7rrr layer_inputrrrrZ self_outputsattention_outputrr8r8r9rUs  zTFMobileBertAttention.callNcC|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurUt|jj|jdWddS1sNwYdSdS)NTr7r)rXrYr3rZr7rCr[rr]r8r8r9r["zTFMobileBertAttention.buildrrR)r;r<r=rErrUr[rar8r8rPr9rs   rcr) TFOutputBottleneckc sZtjdi|tjj|jdd|_t|j|j|j dd|_ tj |j |_ ||_dSNrArBrqrrr8)rDrErrFrGrurArxryrzrqr{r|r}rMrNrPr8r9rEs  zTFOutputBottleneck.__init__FcCs*||}|j||d}|||}|Sr)rAr}rq)r7rTrr layer_outputsr8r8r9rUs zTFOutputBottleneck.callNcCrrrr]r8r8r9r[rzTFOutputBottleneck.buildrrRr`r8r8rPr9rs  rcr) TFMobileBertOutputc stjdi||j|_tjj|jt|jdd|_ t |j |j|j dd|_ |js4tj|j|_nt|dd|_||_dS)NrArrqrr bottleneckrBr8)rDrErrrFrGr\rrwrArxryrzrqr{r|r}rrrMrNrPr8r9rEs  zTFMobileBertOutput.__init__FcCsN||}|js|j||d}|||}|S|||}|||}|Sr)rArr}rqr)r7rTZresidual_tensor_1Zresidual_tensor_2rr8r8r9rUs  zTFMobileBertOutput.callNcCs|jrdSd|_t|dddur2t|jj|jdd|jjgWdn1s-wYt|dddurWt|j j|j dWdn1sRwYt|dddurt|j j|j dWddS1sxwYdSdS)NTrArqr) rXrYr3rZrArCr[rMrHrqrr]r8r8r9r[s "zTFMobileBertOutput.buildrrRr`r8r8rPr9rs   rcr?) TFBottleneckLayerc Jtjdi|tjj|jdd|_t|j|j|j dd|_ ||_ dSr) rDrErrFrGZintra_bottleneck_sizerArxryrzrqrMrNrPr8r9rE   zTFBottleneckLayer.__init__cCs||}||}|SrRrArq)r7rmrTr8r8r9rU s  zTFBottleneckLayer.callNcCrr rXrYr3rZrArCr[rMrurqr]r8r8r9r[rzTFBottleneckLayer.buildrRr`r8r8rPr9r rcr?) TFBottleneckc sLtjdi||j|_|j|_t|dd|_|jr$t|dd|_dSdS)NinputrB attentionr8)rDrEkey_query_shared_bottleneckrrbottleneck_inputrrNrPr8r9rEszTFBottleneck.__init__cCsB||}|jr |fdS|jr||}||||fS||||fS)N)rrrr)r7rTZbottlenecked_hidden_statesZshared_attention_inputr8r8r9rU's     zTFBottleneck.callNcCr)NTrr)rXrYr3rZrrCr[rr]r8r8r9r[ArzTFBottleneck.buildrRr`r8r8rPr9rs rcr?) TFFFNOutputc rr) rDrErrFrGr\rArxryrzrqrMrNrPr8r9rENrzTFFFNOutput.__init__cCs||}|||}|SrRr)r7rTrr8r8r9rUVs zTFFFNOutput.callNcCrr) rXrYr3rZrArCr[rMrHrqr]r8r8r9r[[rzTFFFNOutput.buildrRr`r8r8rPr9rMrrcr?) TFFFNLayerc r)N intermediaterBrr8)rDrEr@rrrrNrPr8r9rEhrzTFFFNLayer.__init__cCs||}|||}|SrR)rr)r7rTintermediate_outputrr8r8r9rUms  zTFFFNLayer.callNcCr)NTrr)rXrYr3rZrrCr[rr]r8r8r9r[rrzTFFFNLayer.buildrRr`r8r8rPr9rgs rcr) TFMobileBertLayerc stjd i|j|_j|_tdd|_tdd|_tdd|_ |jr0t dd|_ jdkrFfddt jdD|_ dSdS) NrrBrrrr&cg|] }td|dqS)zffn.rB)r.0irMr8r9 z.TFMobileBertLayer.__init__..r8)rDrErnum_feedforward_networksrrr@rrrrrrffnrNrPrr9rEs "zTFMobileBertLayer.__init__Fc Cs|jr ||\}}}} n |gd\}}}} |j|||| ||||d} | d} | f} |jdkrBt|jD] \} }|| } | | f7} q4|| }|j|| ||d}|f| ddt d|||| | |f| }|S)Nrrrr&) rrrr enumeraterrrr3constant)r7rTrrrrrrrrZattention_outputsrsrZ ffn_modulerZ layer_outputrr8r8r9rUsJ      zTFMobileBertLayer.callNc Cs|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|dddurwt|jj|jdWdn1srwYt|dddurt|j j|j dWdn1swYt|dddur|j D]}t|j |dWdn1swYqdSdS)NTrrrrr) rXrYr3rZrrCr[rrrrr7r^layerr8r8r9r[s6  zTFMobileBertLayer.buildrrRr`r8r8rPr9r~s  -rcs2eZdZfddZ d ddZd ddZZS) TFMobileBertEncoderc s@tjdi|j|_j|_fddtjD|_dS)Ncr)zlayer_._rB)rrrr8r9rrz0TFMobileBertEncoder.__init__..r8)rDrEroutput_hidden_statesrnum_hidden_layersrrNrPrr9rEszTFMobileBertEncoder.__init__Fc Cs|rdnd}|r dnd} t|jD]#\} } |r||f}| |||| ||d} | d}|r4| | df} q|r<||f}|sJtdd||| fDSt||| dS)Nr8rrr&css|] }|dur|VqdSrRr8)rvr8r8r9 sz+TFMobileBertEncoder.call..)last_hidden_staterT attentions)rrtupler ) r7rTrrrr return_dictrZall_hidden_statesZall_attentionsrZ layer_modulerr8r8r9rUs&   zTFMobileBertEncoder.callNc Csj|jrdSd|_t|dddur1|jD]}t|j |dWdn1s+wYqdSdS)NTr)rXrYrr3rZrCr[rr8r8r9r[s  zTFMobileBertEncoder.buildrrRr`r8r8rPr9rs   #rcr?) TFMobileBertPoolerc sHtjdi||j|_|jrtjj|jt|j ddd|_ ||_ dS)NtanhrA)rZ activationrCr8) rDrEZclassifier_activation do_activaterrFrGrurrwrArMrNrPr8r9rEs zTFMobileBertPooler.__init__cCs(|dddf}|js |S||}|SNr)rrA)r7rTZfirst_token_tensor pooled_outputr8r8r9rUs  zTFMobileBertPooler.callNcCrVrW) rXrYr3rZrArCr[rMrur]r8r8r9r[r_zTFMobileBertPooler.buildrRr`r8r8rPr9rs   rcr?) #TFMobileBertPredictionHeadTransformc srtjdi|tjj|jt|jdd|_t |j t r$t |j |_ n|j |_ td|j|jdd|_||_dS)NrArrnrqrrr8)rDrErrFrGrurrwrArIrJrKrtransform_act_fnrxrzrqrMrNrPr8r9rE$s  z,TFMobileBertPredictionHeadTransform.__init__cCs"||}||}||}|SrR)rArrqrSr8r8r9rU0s   z(TFMobileBertPredictionHeadTransform.callNcCrrrr]r8r8r9r[6rz)TFMobileBertPredictionHeadTransform.buildrRr`r8r8rPr9r#s  rcsNeZdZfddZdddZddZdd Zd d Zd d ZddZ Z S)TFMobileBertLMPredictionHeadc s*tjdi|t|dd|_||_dS)N transformrBr8)rDrErrrMrNrPr8r9rECs z%TFMobileBertLMPredictionHead.__init__NcCs|j|jjfdddd|_|j|jj|jj|jjfdddd|_|j|jj|jjfdddd|_|jr6dSd|_t |dddurat |j j |j dWddS1sZwYdSdS)NrhTrg)rirjZ trainablerCz dense/weightzdecoder/weightr)rlrMrrgrurtrAdecoderrXrYr3rZrrCr[r]r8r8r9r[Hs*"z"TFMobileBertLMPredictionHead.buildcCs|SrRr8r7r8r8r9get_output_embeddings^rz2TFMobileBertLMPredictionHead.get_output_embeddingscCs||_t|d|j_dSr)rrrMrr7rr8r8r9set_output_embeddingsasz2TFMobileBertLMPredictionHead.set_output_embeddingscCs d|jiS)Nrg)rgrr8r8r9get_biases z%TFMobileBertLMPredictionHead.get_biascCs"|d|_t|dd|j_dS)Nrgr)rgrrMrrr8r8r9set_biashs z%TFMobileBertLMPredictionHead.set_biascCs<||}t|tjt|j|jgdd}||j}|S)Nrr)rr3rrrrrArgrSr8r8r9rUls $ z!TFMobileBertLMPredictionHead.callrR) r;r<r=rEr[rrrr rUrar8r8rPr9rBs  rcr?) TFMobileBertMLMHeadc s$tjdi|t|dd|_dS)N predictionsrBr8)rDrErr rNrPr8r9rEtszTFMobileBertMLMHead.__init__cC||}|SrRr )r7sequence_outputprediction_scoresr8r8r9rUx zTFMobileBertMLMHead.callNcCd|jrdSd|_t|dddur0t|jj|jdWddS1s)wYdSdS)NTr )rXrYr3rZr rCr[r]r8r8r9r[|"zTFMobileBertMLMHead.buildrRr`r8r8rPr9r ss r csfeZdZeZdfdd ZddZddZdd Ze dd d Z dddZ Z S)TFMobileBertMainLayerTc sttjdi|||_|j|_|j|_|j|_|j|_t|dd|_ t |dd|_ |r5t |dd|_ dSd|_ dS)NrrBencoderpoolerr8)rDrErMrrrZuse_return_dictrrorrrrr)r7rMadd_pooling_layerrOrPr8r9rEs zTFMobileBertMainLayer.__init__cCs|jSrR)rrr8r8r9get_input_embeddingssz*TFMobileBertMainLayer.get_input_embeddingscCs||j_t|d|j_dSr)rrkrrrr8r8r9set_input_embeddingssz*TFMobileBertMainLayer.set_input_embeddingscCr)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel r)r7Zheads_to_pruner8r8r9 _prune_headssz"TFMobileBertMainLayer._prune_headsNFc  Csd|dur |dur td|durt|} n|dur"t|dd} ntd|dur0t| d}|dur:t| d}|j||||| d} t|| ddd| df} tj| | jd} tjd| jd}tjd | jd}t t || |} |durzt dg|j }|j | | |||| | d}|d}|jdur||nd}| s||f|ddSt|||j|jd S) NzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embedsr&rrr/g?g)rZ pooler_outputrTr)rrr3rrr6r5r0rmultiplysubtractrrrrr rTr)r7rrrrrrrrrrr^Zembedding_outputZextended_attention_maskZone_cstZten_thousand_cstZencoder_outputsrrr8r8r9rUsV      zTFMobileBertMainLayer.callcC|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurRt|jj|jdWdn1sMwYt|dddurzt|jj|jdWddS1sswYdSdS)NTrrr) rXrYr3rZrrCr[rrr]r8r8r9r[ "zTFMobileBertMainLayer.build)T NNNNNNNNNFrR) r;r<r=r' config_classrErrrrrUr[rar8r8rPr9rs&  Src@seZdZdZeZdZdS)TFMobileBertPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mobilebertN)r;r<r=r>r'rZbase_model_prefixr8r8r8r9r sr c@sNeZdZUdZdZded<dZded<dZded<dZded <dZ ded <dS) TFMobileBertForPreTrainingOutputaE Output type of [`TFMobileBertForPreTraining`]. Args: prediction_logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). seq_relationship_logits (`tf.Tensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nztf.Tensor | NonelosszOptional[tf.Tensor]prediction_logitsseq_relationship_logitszTuple[tf.Tensor] | NonerTr) r;r<r=r>r#__annotations__r$r%rTrr8r8r8r9r"s     r"a This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`MobileBertConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). zdThe bare MobileBert Model transformer outputting raw hidden-states without any specific head on top.csfeZdZfddZeeedee e e d          ddddZ dddZ ZS)TFMobileBertModelcs,tj|g|Ri|t|dd|_dS)Nr!rB)rDrErr!r7rMrmrOrPr8r9rEszTFMobileBertModel.__init__batch_size, sequence_length checkpoint output_typerNFrTFModelInputType | Nonernp.ndarray | tf.Tensor | NonerrrrrOptional[bool]rrrr.*Union[Tuple, TFBaseModelOutputWithPooling]c Cs"|j||||||||| | d } | S)N) rrrrrrrrrr)r!) r7rrrrrrrrrrrr8r8r9rUs zTFMobileBertModel.callcCr)NTr!)rXrYr3rZr!rCr[r]r8r8r9r[rzTFMobileBertModel.buildr)rr-rr.rr.rr.rr.rr.rr/rr/rr/rr/r.r0rR)r;r<r=rErr#MOBILEBERT_INPUTS_DOCSTRINGformatr!_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrUr[rar8r8rPr9r's*  r'z MobileBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. cseZdZfddZddZddZeee de e e d d"d#ddZ d$ddZd d!ZZS)%TFMobileBertForPreTrainingcsHtj|g|Ri|t|dd|_t|dd|_t|dd|_dS)Nr!rBpredictions___clsseq_relationship___cls)rDrErr!r r TFMobileBertOnlyNSPHeadseq_relationshipr(rPr8r9rEsz#TFMobileBertForPreTraining.__init__cC|jjSrRr rr8r8r9 get_lm_headz&TFMobileBertForPreTraining.get_lm_headcCs,tdt|jd|jjd|jjjSNzMThe method get_prefix_bias_name is deprecated. Please use `get_bias` instead./)warningswarn FutureWarningrCr rr8r8r9get_prefix_bias_name  z/TFMobileBertForPreTraining.get_prefix_bias_namer)r,rNFrr-rr.rrrrrr/rrr+r1rr..Union[Tuple, TFMobileBertForPreTrainingOutput]c  Cs|j||||||||| | d } | dd\}}||}||}d}| dur<| dur>> import tensorflow as tf >>> from transformers import AutoTokenizer, TFMobileBertForPreTraining >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = TFMobileBertForPreTraining.from_pretrained("google/mobilebert-uncased") >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 >>> outputs = model(input_ids) >>> prediction_scores, seq_relationship_scores = outputs[:2] ``` rrrrrrrrrNrr+r1r+r-)r#r$r%rTr)r!r r9r:r"rTr)r7rrrrrrrrrr+r1rrrrrseq_relationship_scoreZ total_lossZd_labelsrr8r8r9rUs<!  zTFMobileBertForPreTraining.callcCr)NTr!r r9) rXrYr3rZr!rCr[r r9r]r8r8r9r[ rz TFMobileBertForPreTraining.buildcC|dkr|dfS|fSNzcls.predictions.decoder.weightz,mobilebert.embeddings.word_embeddings.weightr8r7Z tf_weightr8r8r9tf_to_pt_weight_rename.z1TFMobileBertForPreTraining.tf_to_pt_weight_rename NNNNNNNNNNNF)rr-rr.rr.rr.rr.rr.rr/rr/rr/r+r.r1r.rr/r.rErR)r;r<r=rEr;rBrr#r1r2r%r"r4rUr[rLrar8r8rPr9r5s,    Ar5z8MobileBert Model with a `language modeling` head on top.c seZdZgdZfddZddZddZeee de e e ed d d  d$d%ddZd&d d!Zd"d#ZZS)'TFMobileBertForMaskedLM)rr7cls.seq_relationshipcs<tj|g|Ri|t|ddd|_t|dd|_dS)NFr!rrCr6rB)rDrErr!r r r(rPr8r9rE>sz TFMobileBertForMaskedLM.__init__cCr:rRr rr8r8r9r;Dr<z#TFMobileBertForMaskedLM.get_lm_headcCs,tdt|jd|jjd|jjjSr=)r?r@rArCZmlmr rr8r8r9rBGrCz,TFMobileBertForMaskedLM.get_prefix_bias_namer)z'paris'g= ףp=?r+r,rexpected_output expected_lossNFrr-rr.rrrrrr/rrr+rr.Union[Tuple, TFMaskedLMOutput]c  Cs|j||||||||| | d } | d} |j| | d}| dur dn|| |}| s<|f| dd}|dur:|f|S|St||| j| jdS)az labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels rFrrNrr#r-rTr)r!r r:r rTr)r7rrrrrrrrrr+rrrrr#rr8r8r9rUKs0 zTFMobileBertForMaskedLM.callcCr)NTr!r )rXrYr3rZr!rCr[r r]r8r8r9r[rzTFMobileBertForMaskedLM.buildcCrIrJr8rKr8r8r9rLrMz.TFMobileBertForMaskedLM.tf_to_pt_weight_rename NNNNNNNNNNF)rr-rr.rr.rr.rr.rr.rr/rr/rr/r+r.rr/r.rUrR)r;r<r="_keys_to_ignore_on_load_unexpectedrEr;rBrr#r1r2r!r3r r4rUr[rLrar8r8rPr9rO5s8    0 rOcr?) r8c s.tjdi|tjjddd|_||_dS)Nrr9rBr8)rDrErrFrGr9rMrNrPr8r9rEs z TFMobileBertOnlyNSPHead.__init__cCr rR)r9)r7rrHr8r8r9rUrzTFMobileBertOnlyNSPHead.callNcCrV)NTr9) rXrYr3rZr9rCr[rMrur]r8r8r9r[r_zTFMobileBertOnlyNSPHead.buildrRr`r8r8rPr9r8s r8zPMobileBert Model with a `next sentence prediction (classification)` head on top.csneZdZddgZfddZeeede e e d           ddddZ dddZ ZS) %TFMobileBertForNextSentencePredictionr6cls.predictionscs:tj|g|Ri|t|dd|_t|dd|_dS)Nr!rBr7)rDrErr!r8clsr(rPr8r9rEsz.TFMobileBertForNextSentencePrediction.__init__r)rDNFrr-rr.rrrrrr/rrr1rr.+Union[Tuple, TFNextSentencePredictorOutput]c  Cs|j||||||||| | d } | d} || }| durdn|j| |d}| s;|f| dd}|dur9|f|S|St||| j| jdS)a Return: Examples: ```python >>> import tensorflow as tf >>> from transformers import AutoTokenizer, TFMobileBertForNextSentencePrediction >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = TFMobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="tf") >>> logits = model(encoding["input_ids"], token_type_ids=encoding["token_type_ids"])[0] ```rFr&NrGrrV)r!r[r:r rTr)r7rrrrrrrrrr1rrrZseq_relationship_scoresZnext_sentence_lossrr8r8r9rUs6#  z*TFMobileBertForNextSentencePrediction.callcCr)NTr!r[)rXrYr3rZr!rCr[r[r]r8r8r9r[rz+TFMobileBertForNextSentencePrediction.buildrW)rr-rr.rr.rr.rr.rr.rr/rr/rr/r1r.rr/r.r\rR)r;r<r=rXrErr#r1r2r%r r4rUr[rar8r8rPr9rYs&   @rYz MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c zeZdZgdZdgZfddZeee de e e e eed           ddddZdddZZS) %TFMobileBertForSequenceClassificationr6r7rZrPr}csztj|g|Ri||j|_t|dd|_|jdur |jn|j}tj ||_ tjj |jt |j dd|_||_dS)Nr!rB classifierrrDrE num_labelsrr!classifier_dropoutr|rrFr{r}rGrrwr`rMr7rMrmrOrcrPr8r9rEs z.TFMobileBertForSequenceClassification.__init__r)rRNFrr-rr.rrrrrr/rrr+rr.(Union[Tuple, TFSequenceClassifierOutput]c  C|j||||||||| | d } | d} |j| | d} || }| dur%dn|| |}| sA|f| dd}|dur?|f|S|St||| j| jdS)a labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). rFr&rNrrV)r!r}r`r:rrTr)r7rrrrrrrrrr+rrrr-r#rr8r8r9rU%s2  z*TFMobileBertForSequenceClassification.callcC|jrdSd|_t|dddur-t|jj|jdWdn1s(wYt|dddurZt|jj|jdd|jj gWddS1sSwYdSdSNTr!r` rXrYr3rZr!rCr[r`rMrur]r8r8r9r[`"z+TFMobileBertForSequenceClassification.buildrW)rr-rr.rr.rr.rr.rr.rr/rr/rr/r+r.rr/r.rerR)r;r<r=rX_keys_to_ignore_on_load_missingrErr#r1r2r!'_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATIONrr4_SEQ_CLASS_EXPECTED_OUTPUT_SEQ_CLASS_EXPECTED_LOSSrUr[rar8r8rPr9r^s4    2r^z MobileBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). c szeZdZgdZfddZeeede e e e e eeed            ddddZdddZZS) TFMobileBertForQuestionAnsweringrr6r7rZrPcsXtj|g|Ri||j|_t|ddd|_tjj|jt|j dd|_ ||_ dS)NFr!rQ qa_outputsr) rDrErbrr!rrFrGrrwrqrMr(rPr8r9rE}s z)TFMobileBertForQuestionAnswering.__init__r))r+r,rZqa_target_start_indexZqa_target_end_indexrSrTNFrr-rr.rrrrrr/rrstart_positions end_positionsrr.,Union[Tuple, TFQuestionAnsweringModelOutput]c  Cs|j||||||||| | d } | d}||}tj|ddd\}}tj|dd}tj|dd}d}| durG| durG| | d}||||f}| s^||f| dd}|dur\|f|S|St|||| j| jdS) a start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. rFrrrrN)Zstart_positionZ end_position)r# start_logits end_logitsrTr) r!rqr3splitZsqueezer:rrTr)r7rrrrrrrrrrrrsrrrr-rurvr#r+rr8r8r9rUs>$  z%TFMobileBertForQuestionAnswering.callcCrg)NTr!rq) rXrYr3rZr!rCr[rqrMrur]r8r8r9r[rjz&TFMobileBertForQuestionAnswering.buildrN)rr-rr.rr.rr.rr.rr.rr/rr/rr/rrr.rsr.rr/r.rtrR)r;r<r=rXrErr#r1r2r!_CHECKPOINT_FOR_QArr4_QA_TARGET_START_INDEX_QA_TARGET_END_INDEX_QA_EXPECTED_OUTPUT_QA_EXPECTED_LOSSrUr[rar8r8rPr9rols8   =roz MobileBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. csveZdZgdZdgZfddZeee de e e e d           ddddZdddZZS) TFMobileBertForMultipleChoicer_r}cs\tj|g|Ri|t|dd|_tj|j|_tjj dt |j dd|_ ||_ dS)Nr!rBr&r`r)rDrErr!rrFr{r|r}rGrrwr`rMr(rPr8r9rEs  z&TFMobileBertForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr*NFrr-rr.rrrrrr/rrr+rr.)Union[Tuple, TFMultipleChoiceModelOutput]c  Csv|durt|d} t|d} n t|d} t|d} |dur)t|d| fnd}|dur7t|d| fnd}|durEt|d| fnd}|durSt|d| fnd}|durft|d| t|dfnd}|j||||||||| | d }|d}|j|| d}||}t|d| f}| durdn|| |}| s|f|dd}|dur|f|S|St|||j|j dS) a5 labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr&rrr)rrrrV) rr3r6r!r}r`r:r rTr)r7rrrrrrrrrr+rZ num_choicesZ seq_lengthZflat_input_idsZflat_attention_maskZflat_token_type_idsZflat_position_idsZflat_inputs_embedsrrr-Zreshaped_logitsr#rr8r8r9rUsN     z"TFMobileBertForMultipleChoice.callcCrgrhrir]r8r8r9r[@rjz#TFMobileBertForMultipleChoice.buildrW)rr-rr.rr.rr.rr.rr.rr/rr/rr/r+r.rr/r.r~rR)r;r<r=rXrkrErr#r1r2r!r3r r4rUr[rar8r8rPr9r}s4   Ar}z MobileBert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. c r]) "TFMobileBertForTokenClassificationrpr}cs|tj|g|Ri||j|_t|ddd|_|jdur!|jn|j}tj ||_ tjj |jt |j dd|_||_dS)NFr!rQr`rrardrPr8r9rE^s z+TFMobileBertForTokenClassification.__init__r)rRNFrr-rr.rrrrrr/rrr+rr.%Union[Tuple, TFTokenClassifierOutput]c  Crf)z labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. rFrrNrrV)r!r}r`r:rrTr)r7rrrrrrrrrr+rrrr-r#rr8r8r9rUls2  z'TFMobileBertForTokenClassification.callcCrgrhrir]r8r8r9r[rjz(TFMobileBertForTokenClassification.buildrW)rr-rr.rr.rr.rr.rr.rr/rr/rr/r+r.rr/r.rrR)r;r<r=rXrkrErr#r1r2r!$_CHECKPOINT_FOR_TOKEN_CLASSIFICATIONrr4_TOKEN_CLASS_EXPECTED_OUTPUT_TOKEN_CLASS_EXPECTED_LOSSrUr[rar8r8rPr9rLs4    0r) rOr}rYr5ror^rrr'r )jr> __future__rr? dataclassesrtypingrrrnumpynpZ tensorflowr3Zactivations_tfrZmodeling_tf_outputsr r r r r rrrZmodeling_tf_utilsrrrrrrrrrrrrZtf_utilsrrrutilsr r!r"r#r$r%Zconfiguration_mobilebertr'Z get_loggerr;loggerr3r4rrrrxr{r|ryrzrlrmrnr*rFZLayerr@ZLayerNormalizationrbrerxrorrrrrrrrrrrrrrr rr r"ZMOBILEBERT_START_DOCSTRINGr1r'r5rOr8rYr^ror}r__all__r8r8r8r9s   (8     he!()/Q4 1 *81g`Y_hj ^