o Zh@@s~dZddlZddlZddlmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZddlmZdd lmZmZmZmZmZmZmZmZdd lmZdd lmZmZm Z dd l!m"Z"m#Z#d dl$m%Z%e#&e'Z(ddZ)Gddde j*Z+Gddde j*Z,Gddde j*Z-Gddde j*Z.Gddde j*Z/Gddde j*Z0Gddde j*Z1Gdd d e j*Z2Gd!d"d"e j*Z3Gd#d$d$e j*Z4Gd%d&d&e j*Z5Gd'd(d(e j*Z6e"Gd)d*d*eZ7e"d+d,Gd-d.d.e7Z8e"Gd/d0d0e7Z9e"d1d,Gd2d3d3e7eZ:e"d4d,Gd5d6d6e7Z;e"Gd7d8d8e7Zgd=Z?dS)>zPyTorch RemBERT model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)GenerationMixin))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging) RemBertConfigc szddl}ddl}ddl}Wn tytdwtj|}t d||j |}g}g} |D],\} t fdddDrFq6t dd | |j |} || | q6t|| D]\} d d d t d dDrt dd qh|} D]}} |d| r|d| }n| g}|ddks|ddkrt| d} nH|ddks|ddkrt| d} n6|ddkrt| d} n*|ddkrt| d} nz t| |d} Wntyt dd Yqwt|dkr t|d}| |} q| dddkrt| d} n | dkr'|| } z| j| jkrEsz-load_tf_weights_in_rembert..)adam_vadam_mZoutput_embeddingclszLoading TF weight z with shape zbert/zrembert//css|]}|dvVqdS))r#r$ZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Z global_stepNr)rnrrr!r"Vs  z Skipping z [A-Za-z]+_\d+z_(\d+)ZkernelgammaweightZ output_biasbetabiasZoutput_weightsZsquad classifierz Skipping {}riZ _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )!renumpyZ tensorflow ImportErrorloggererrorospathabspathinfotrainZlist_variablesanyZ load_variableappendzipreplacesplitjoin fullmatchgetattrAttributeErrorformatlenint transposeshape ValueErrorAssertionErrorargstorchZ from_numpydata)modelconfigZtf_checkpoint_pathr.nptfZtf_pathZ init_varsnamesZarraysrEarrayZpointerZm_nameZ scope_namesnumerrr!load_tf_weights_in_rembert/s                     rSc sheZdZdZfddZ     ddeejdeejdeejd eejd e d ej f d d Z Z S)RemBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_|jdt|jddddS)N) padding_idxZeps position_ids)rF) persistent)super__init__r Embedding vocab_sizeinput_embedding_size pad_token_idword_embeddingsZmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutZregister_bufferrIZarangeexpandselfrL __class__rr!r[s   zRemBertEmbeddings.__init__Nr input_idstoken_type_idsrW inputs_embedspast_key_values_lengthreturnc Cs|dur |}n|dd}|d}|dur&|jdd|||f}|dur5tj|tj|jjd}|dur>||}||}||} ||} | | 7} | | } | | } | S)NrXrdtypedevice) sizerWrIzeroslongrtr`rbrarcrg) rjrmrnrWrorp input_shape seq_lengthrb embeddingsrarrr!forwards"      zRemBertEmbeddings.forward)NNNNr) __name__ __module__ __qualname____doc__r[rrI LongTensor FloatTensorrCTensorr{ __classcell__rrrkr!rTs* rTc2eZdZfddZdejdejfddZZS) RemBertPoolercs*tt|j|j|_t|_dSr)rZr[rLinear hidden_sizedenseZTanh activationrirkrr!r[s zRemBertPooler.__init__ hidden_statesrqcCs(|dddf}||}||}|S)Nr)rr)rjrZfirst_token_tensor pooled_outputrrr!r{s  zRemBertPooler.forwardr|r}r~r[rIrr{rrrrkr!rs rcseZdZfddZddZ      ddejdeejd eejd eejd eejd ee e ejd e de fddZ Z S)RemBertSelfAttentioncst|j|jdkrt|dstd|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|j|_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ())rZr[rnum_attention_headshasattrrFrCattention_head_size all_head_sizerrquerykeyvaluereZattention_probs_dropout_probrg is_decoderrirkrr!r[s   zRemBertSelfAttention.__init__cCs6|dd|j|jf}|j|}|ddddS)NrXrr-rr )rurrviewpermute)rjxZ new_x_shaperrr!transpose_for_scoress z)RemBertSelfAttention.transpose_for_scoresNFrattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsrqcCs||}|du} | r|dur|d} |d} |}nP| r/|||} |||} |}n;|durZ|||} |||} tj|d| gdd} tj|d| gdd} n|||} |||} ||} |jrv| | f}t| | dd} | t |j } |dur| |} t j j| dd}||}|dur||}t|| }|dddd}|dd|jf}|j|}|r||fn|f}|jr||f}|S)Nrrr-dimrXr )rrrrrIcatrmatmulrDmathsqrtrrZ functionalZsoftmaxrgr contiguousrurr)rjrrrrrrrZmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layerZattention_scoresZattention_probsZ context_layerZnew_context_layer_shapeoutputsrrr!r{sH       zRemBertSelfAttention.forwardNNNNNF) r|r}r~r[rrIrrrrboolr{rrrrkr!rs4  rc8eZdZfddZdejdejdejfddZZS)RemBertSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNrV) rZr[rrrrrcrdrerfrgrirkrr!r[3 zRemBertSelfOutput.__init__r input_tensorrqcC&||}||}|||}|Srrrgrcrjrrrrr!r{9  zRemBertSelfOutput.forwardrrrrkr!r2 $rcseZdZfddZddZ      ddejdeejd eejd eejd eejd ee e ejd ee de ejfddZ Z S)RemBertAttentioncs*tt||_t||_t|_dSr)rZr[rrjroutputset pruned_headsrirkrr!r[As    zRemBertAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr)rBrrjrrrrrrrrrrunion)rjheadsindexrrr! prune_headsHs zRemBertAttention.prune_headsNFrrrrrrrrqc Cs<||||||||}||d|} | f|dd} | S)Nrr)rjr) rjrrrrrrrZ self_outputsattention_outputrrrr!r{[s  zRemBertAttention.forwardr) r|r}r~r[rrIrrrrrr{rrrrkr!r@s4  rcr)RemBertIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) rZr[rrrintermediate_sizer isinstance hidden_actstrr intermediate_act_fnrirkrr!r[us   zRemBertIntermediate.__init__rrqcCs||}||}|Sr)rrrjrrrr!r{}s  zRemBertIntermediate.forwardrrrrkr!rts rcr) RemBertOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) rZr[rrrrrrcrdrerfrgrirkrr!r[rzRemBertOutput.__init__rrrqcCrrrrrrr!r{rzRemBertOutput.forwardrrrrkr!rrrcseZdZfddZ      ddejdeejdeejdeejd eejd eeeejd ee d eejfd dZ ddZ Z S) RemBertLayercsnt|j|_d|_t||_|j|_|j|_|jr+|js&t|dt||_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is added)rZr[chunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrFcrossattentionr intermediaterrrirkrr!r[s    zRemBertLayer.__init__NFrrrrrrrrqc Cs|dur |ddnd}|j|||||d} | d} |jr(| dd} | d} n| dd} d} |jro|durot|dsDtd|d|durN|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nr-)rrrrrXrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrrrFrrfeed_forward_chunkrr)rjrrrrrrrZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputs layer_outputrrr!r{sP         zRemBertLayer.forwardcCs||}|||}|Sr)rr)rjrZintermediate_outputrrrr!rs  zRemBertLayer.feed_forward_chunkr) r|r}r~r[rIrrrrrr{rrrrrkr!rs4  BrcseZdZfddZ         ddejdeejdeejd eejd eejd eeeejd ee d e de de de ee ffddZ Z S)RemBertEncodercsLt|_tjj|_tfddt j D|_ d|_ dS)Ncsg|]}tqSr)r)r_rLrr! sz+RemBertEncoder.__init__..F) rZr[rLrrr^rembedding_hidden_mapping_inZ ModuleListrangenum_hidden_layerslayergradient_checkpointingrirkrr!r[s   zRemBertEncoder.__init__NFTrrrrrpast_key_values use_cacheroutput_hidden_states return_dictrqc  Csh|jr|jr|rtdd}||}| rdnd} |rdnd} |r(|jjr(dnd} |r.dnd}t|jD]^\}}| r@| |f} |durH||nd}|durR||nd}|jrh|jrh| |j |||||||}n ||||||||}|d}|r||df7}|r| |df} |jjr| |df} q5| r| |f} | st dd ||| | | fDSt ||| | | d S) NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FrrrXrr-css|] }|dur|VqdSrr)rvrrr!r"5sz)RemBertEncoder.forward..)last_hidden_staterr attentionscross_attentions) rZtrainingr1Z warning_oncerrLr enumeraterZ_gradient_checkpointing_func__call__tupler )rjrrrrrrrrrrZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZ layer_moduleZlayer_head_maskrZ layer_outputsrrr!r{s|           zRemBertEncoder.forward) NNNNNNFFT)r|r}r~r[rIrrrrrrr r{rrrrkr!rsD      rcr)RemBertPredictionHeadTransformcsVtt|j|j|_t|jtrt |j|_ n|j|_ tj |j|j d|_ dSr) rZr[rrrrrrrr transform_act_fnrcrdrirkrr!r[Ks  z'RemBertPredictionHeadTransform.__init__rrqcCs"||}||}||}|Sr)rrrcrrrr!r{Ts   z&RemBertPredictionHeadTransform.forwardrrrrkr!rJs  rcr)RemBertLMPredictionHeadcsRtt|j|j|_t|j|j|_t |j |_ tj |j|j d|_ dSr)rZr[rrrZoutput_embedding_sizerr]decoderr rrrcrdrirkrr!r[\s  z RemBertLMPredictionHead.__init__rrqcCs,||}||}||}||}|Sr)rrrcrrrrr!r{cs    zRemBertLMPredictionHead.forwardrrrrkr!r[s rcr)RemBertOnlyMLMHeadcstt||_dSr)rZr[r predictionsrirkrr!r[ms zRemBertOnlyMLMHead.__init__sequence_outputrqcCs||}|Sr)r)rjrprediction_scoresrrr!r{qs zRemBertOnlyMLMHead.forwardrrrrkr!rls rc@s$eZdZeZeZdZdZddZ dS)RemBertPreTrainedModelrembertTcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSdS)zInitialize the weightsg)meanZstdNg?)rrrr)rJZnormal_rLZinitializer_ranger+Zzero_r\rUrcZfill_)rjmodulerrr! _init_weights}s      z$RemBertPreTrainedModel._init_weightsN) r|r}r~rZ config_classrSZload_tf_weightsZbase_model_prefixZsupports_gradient_checkpointingrrrrr!rvs  ra The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. )Z custom_introc seZdZdfdd ZddZddZdd Ze dd ee j d ee j d ee j dee j dee j dee j dee j dee j dee e e j dee dee dee dee dee effddZZS) RemBertModelTcsDt|||_t||_t||_|rt|nd|_| dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) rZr[rLrTrzrencoderrpooler post_init)rjrLadd_pooling_layerrkrr!r[s    zRemBertModel.__init__cCs|jjSrrzr`rjrrr!get_input_embeddingssz!RemBertModel.get_input_embeddingscCs ||j_dSrr)rjrrrr!set_input_embeddingss z!RemBertModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)rjZheads_to_prunerrrrr! _prune_headsszRemBertModel._prune_headsNrmrrnrWrrorrrrrrrrqc Cs| dur| n|jj} | dur| n|jj} | dur| n|jj} |jjr-| dur(| n|jj} nd} |dur;|dur;td|durJ||||}n|durW|dd}ntd|\}}|durf|j n|j }| durv| ddj dnd}|durt j |||f|d}|durt j |t j|d}|||}|jjr|dur|\}}}||f}|durt j ||d}||}nd}|||jj}|j|||||d }|j|||||| | | | | d }|d}|jdur||nd}| s||f|d dSt|||j|j|j|jd S) NFzDYou cannot specify both input_ids and inputs_embeds at the same timerXz5You have to specify either input_ids or inputs_embedsrr-)rtrr)rmrWrnrorp) rrrrrrrrrr)rZ pooler_outputrrrr)rLrruse_return_dictrrrFZ%warn_if_padding_and_no_attention_maskrurtrErIZonesrvrwZget_extended_attention_maskZinvert_attention_maskZ get_head_maskrrzrrr rrrr)rjrmrrnrWrrorrrrrrrrxZ batch_sizeryrtrpZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsrrrrr!r{s|     zRemBertModel.forward)T) NNNNNNNNNNNNN)r|r}r~r[rrrrrrIrrrrrr r{rrrrkr!rs`       rcseZdZdgZfddZddZddZe            dd ee j d ee j d ee j d ee j d ee j dee j dee j dee j dee j dee dee dee de eeffddZdddZede fddZZS)RemBertForMaskedLMcls.predictions.decoder.weightcs@t||jrtdt|dd|_t||_| dS)NznIf you want to use `RemBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr rZr[rr1warningrrrr%rrirkrr!r['s   zRemBertForMaskedLM.__init__cC |jjjSrr%rrrrrr!get_output_embeddings6 z(RemBertForMaskedLM.get_output_embeddingscC||jj_dSrrrjZnew_embeddingsrrr!set_output_embeddings9z(RemBertForMaskedLM.set_output_embeddingsNrmrrnrWrrorrlabelsrrrrqc  Cs| dur| n|jj} |j||||||||| | | d } | d}||}d}| dur:t}||d|jj| d}| sP|f| dd}|durN|f|S|St||| j| j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. N) rrnrWrrorrrrrrrXr-losslogitsrr) rLrrr%rrr]rrr)rjrmrrnrWrrorrr rrrrrrZmasked_lm_lossloss_fctrrrr!r{<s: zRemBertForMaskedLM.forwardcKs~|j}|d}|jjdusJdtj|||jddfgdd}tj|df|jjtj|jd}tj||gdd}||dS)Nrz.The PAD token should be defined for generationrrXrrr)rmr) rErLr_rIrZ new_zerosfullrwrt)rjrmrZ model_kwargsrxZeffective_batch_sizeZ dummy_tokenrrr!prepare_inputs_for_generationus" z0RemBertForMaskedLM.prepare_inputs_for_generationcCsdS)z Legacy correction: RemBertForMaskedLM can't call `generate()` from `GenerationMixin`, even though it has a `prepare_inputs_for_generation` method. Fr)r%rrr! can_generateszRemBertForMaskedLM.can_generate) NNNNNNNNNNNNr)r|r}r~_tied_weights_keysr[rr rrrIrrrrrrr{r classmethodrrrrrkr!r#s`        8rzS RemBERT Model with a `language modeling` head on top for CLM fine-tuning. c"seZdZdgZfddZddZddZe              dd ee j d ee j d ee j d ee j d ee j dee j dee j dee j dee e e j dee j dee dee dee dee dee effddZddZZS)RemBertForCausalLMrcs@t||jstdt|dd|_t||_| dS)NzOIf you want to use `RemBertForCausalLM` as a standalone, add `is_decoder=True.`Frrrirkrr!r[s    zRemBertForCausalLM.__init__cCrrrrrrr!rrz(RemBertForCausalLM.get_output_embeddingscCrrrrrrr!r r z(RemBertForCausalLM.set_output_embeddingsNrmrrnrWrrorrrr rrrrrqcKs|dur|n|jj}|j||||||||| | | | |d }|d}||}d}| dur9|j|| fd|jji|}|sO|f|dd}|durM|f|S|St|||j|j|j |j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert") >>> config = RemBertConfig.from_pretrained("google/rembert") >>> config.is_decoder = True >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```N) rrnrWrrorrrrrrrrr]r-)r rrrrr) rLrrr%Z loss_functionr]rrrrr)rjrmrrnrWrrorrrr rrrrkwargsrrrZlm_lossrrrr!r{sN) zRemBertForCausalLM.forwardcsBd}|D]}|tfdd|ddD|ddf7}q|S)Nrc3s$|] }|d|jVqdS)rN)Z index_selecttort)rZ past_statebeam_idxrr!r"s"z4RemBertForCausalLM._reorder_cache..r-)r)rjrrZreordered_pastZ layer_pastrrr!_reorder_caches z!RemBertForCausalLM._reorder_cache)NNNNNNNNNNNNNN)r|r}r~rr[rr rrrIrrrrrrr{rrrrrkr!rsh         Srz RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZfddZe          ddeejdeejdeejdeejdeejd eejd eejd ee d ee d ee de e e ffddZ ZS) RemBertForSequenceClassificationcsJt||j|_t||_t|j|_t |j |j|_ | dSr rZr[ num_labelsrrrreclassifier_dropout_probrgrrr,rrirkrr!r[ s   z)RemBertForSequenceClassification.__init__NrmrrnrWrror rrrrqc  Csr| dur| n|jj} |j|||||||| | d } | d} || } || } d}|dur|jjdurV|jdkr 1` a classification loss is computed (Cross-Entropy). NrrnrWrrorrrrZ regressionZsingle_label_classificationZmulti_label_classificationrXr-r )rLrrrgr,Z problem_typerrsrIrwrCrsqueezerrrrrr)rjrmrrnrWrror rrrrrrr rrrrr!r{sV     "       z(RemBertForSequenceClassification.forward NNNNNNNNNN)r|r}r~r[rrrIrrrrrrr{rrrrkr!rsH      rcr)RemBertForMultipleChoicecs@t|t||_t|j|_t|j d|_ | dS)Nr) rZr[rrrrerrgrrr,rrirkrr!r[`s   z!RemBertForMultipleChoice.__init__NrmrrnrWrror rrrrqc  Csn| dur| n|jj} |dur|jdn|jd} |dur%|d|dnd}|dur4|d|dnd}|durC|d|dnd}|durR|d|dnd}|dure|d|d|dnd}|j|||||||| | d } | d} || } || }|d| }d}|durt}|||}| s|f| dd}|dur|f|S|St ||| j | j dS)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) NrrXrr r-r ) rLrrErrurrgr,rrrr)rjrmrrnrWrror rrrZ num_choicesrrrZreshaped_logitsr rrrrr!r{jsL,    z RemBertForMultipleChoice.forwardr")r|r}r~r[rrrIrrrrrrr{rrrrkr!r#^sH      r#cr)RemBertForTokenClassificationcsNt||j|_t|dd|_t|j|_t |j |j|_ | dSNFrrrirkrr!r[s  z&RemBertForTokenClassification.__init__NrmrrnrWrror rrrrqc  Cs| dur| n|jj} |j|||||||| | d } | d} || } || } d}|dursj   (   S7j4X`   hsTgB N