o Zh@sdZddlZddlmZmZmZmZddlZddlZddl m Z ddlm Z ddl m Z mZmZddlmZmZdd lmZdd lmZmZdd lmZmZmZmZmZmZmZm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&ddl'm(Z(m)Z)m*Z*ddl+m,Z,e*-e.Z/Gddde j0Z1Gddde j0Z2Gddde2Z3Gddde j0Z4e2e3dZ5Gddde j0Z6Gddde j0Z7Gddde j0Z8Gd d!d!e j0Z9Gd"d#d#e j0Z:Gd$d%d%e j0Z;e(Gd&d'd'e"Ze(Gd/d0d0e<Z?Gd1d2d2e j0Z@e(d3d)Gd4d5d5e<ZAe(Gd6d7d7e<ZBe(Gd8d9d9e<ZCGd:d;d;e j0ZDe(Gdd?ZFgd@ZGdS)BzPyTorch RoBERTa model.N)ListOptionalTupleUnion)version)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FNgelu)GenerationMixin)#_prepare_4d_attention_mask_for_sdpa*_prepare_4d_causal_attention_mask_for_sdpa))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringget_torch_versionlogging) RobertaConfigcs4eZdZdZfddZ d ddZdd ZZS) RobertaEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_t|dd|_|jdt|jddd|jd tj|jtjd dd|j|_tj|j|j|jd|_ dS) N) padding_idxZepsposition_embedding_typeabsolute position_ids)r F) persistenttoken_type_idsdtype)super__init__r Embedding vocab_size hidden_sizeZ pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutgetattrr%Zregister_buffertorcharangeexpandzerosr'sizelongr#selfconfig __class__[/var/www/auris/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.pyr.7s"   zRobertaEmbeddings.__init__Nrc Cs|dur|durt||j|}n||}|dur|}n|dd}|d}|durTt|drI|jddd|f}||d|} | }n tj|tj |j j d}|dur]| |}| |} || } |jdkrt||} | | 7} || } || } | S)Nr(r r*rr,devicer&)"create_position_ids_from_input_idsr#&create_position_ids_from_inputs_embedsr@hasattrr*r>r<r?rAr'rJr2r5r%r4r6r:) rC input_idsr*r' inputs_embedspast_key_values_length input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr5 embeddingsr4rGrGrHforwardPs0         zRobertaEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr(r rIr)r@r<r=r#rArJZ unsqueezer>)rCrOrQZsequence_lengthr'rGrGrHrLxs  z8RobertaEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)__name__ __module__ __qualname____doc__r.rVrL __classcell__rGrGrErHr"1s   (r"cseZdZdfdd ZdejdejfddZ      dd ejd eejd eejd eejd eejdee e ejdee de ejfddZ Z S)RobertaSelfAttentionNcst|j|jdkrt|dstd|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|p\t|dd|_|jdksh|jd kry|j|_t d |jd |j|_|j|_dS) NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r%r& relative_keyrelative_key_queryr )r-r.r1num_attention_headsrM ValueErrorintattention_head_size all_head_sizerLinearquerykeyvaluer8attention_probs_dropout_probr:r;r%r3r/distance_embedding is_decoderrCrDr%rErGrHr.s*   zRobertaSelfAttention.__init__xreturncCs6|dd|j|jf}||}|ddddS)Nr(rr`r r )r@rardviewpermute)rCrnZ new_x_shaperGrGrHtranspose_for_scoress z)RobertaSelfAttention.transpose_for_scoresF hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionscCs||}|du} | r|dur|d} |d} |}nP| r/|||} |||} |}n;|durZ|||} |||} tj|d| gdd} tj|d| gdd} n|||} |||} ||} |du} |jrz| | f}t| | dd}|j dks|j dkr | j d| j d}}| rtj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n|j dkr td | |}td | |}|||}|t|j}|dur||}tjj|dd}||}|dur0||}t|| }|dddd }|dd|jf}||}|rX||fn|f}|jrd||f}|S)Nrr r`dimr(r^r_rIr+zbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rgrrrhrir<catrlmatmul transposer%shapeZtensorrArJrpr=rkr3tor,Zeinsummathsqrtrdr functionalZsoftmaxr:rq contiguousr@re)rCrsrtrurvrwrxryZmixed_query_layeris_cross_attention key_layer value_layer query_layer use_cacheZattention_scoresZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeoutputsrGrGrHrVsn             zRobertaSelfAttention.forwardNNNNNNF) rWrXrYr.r<Tensorrrr FloatTensorrboolrVr[rGrGrErHr\s4 r\cseZdZdfdd Z      ddejdeejdeejdeejd eejd eeeejd ee d eejffd d Z Z S)RobertaSdpaSelfAttentionNcs4tj||d|j|_tttdk|_dS)Nr%z2.2.0)r-r.rj dropout_probrparserrequire_contiguous_qkvrmrErGrHr.sz!RobertaSdpaSelfAttention.__init__Frsrtrurvrwrxryroc s|jdks |s |durtdt|||||||S|\}} } |||} |du} | r3|n|} | r9|n|}| rP|rP|djd| jdkrP|\}}n,|| | }|| | }|dur|| s|t j |d|gdd}t j |d|gdd}|j r||f}|jr| jjdkr|dur| } |}|}|j r| s|dur| dkrdnd }t jjj| ||||jr|jnd |d }|dd}||| |j}|f}|j r||f}|S) Nr&aRobertaSdpaSelfAttention is used but `torch.nn.functional.scaled_dot_product_attention` does not support non-absolute `position_embedding_type` or `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rr`r rzcudaTF)Z attn_maskZ dropout_p is_causal)r%logger warning_oncer-rVr@rrrgrrhrir<r}rlrrJtyperrrZscaled_dot_product_attentiontrainingrrZreshapere)rCrsrtrurvrwrxryZbsztgt_len_rrZcurrent_statesrrrZ attn_outputrrErGrHrVs^          z RobertaSdpaSelfAttention.forwardrr) rWrXrYr.r<rrrrrrVr[rGrGrErHrs2  rc8eZdZfddZdejdejdejfddZZS)RobertaSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr$) r-r.rrfr1denser6r7r8r9r:rBrErGrHr.y zRobertaSelfOutput.__init__rs input_tensorrocC&||}||}|||}|Srrr:r6rCrsrrGrGrHrV  zRobertaSelfOutput.forwardrWrXrYr.r<rrVr[rGrGrErHrx $r)eagersdpacseZdZdfdd ZddZ      ddejdeejd eejd eejd eejd ee e ejd ee de ejfddZ Z S)RobertaAttentionNcs4tt|j||d|_t||_t|_dS)Nr) r-r.ROBERTA_SELF_ATTENTION_CLASSES_attn_implementationrCroutputset pruned_headsrmrErGrHr.s   zRobertaAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrr rz)lenrrCrardrrrgrhrirrreunion)rCheadsindexrGrGrH prune_headss zRobertaAttention.prune_headsFrsrtrurvrwrxryroc Cs<||||||||}||d|} | f|dd} | S)Nrr )rCr) rCrsrtrurvrwrxryZ self_outputsattention_outputrrGrGrHrVs  zRobertaAttention.forwardrr) rWrXrYr.rr<rrrrrrVr[rGrGrErHrs4 rc2eZdZfddZdejdejfddZZS)RobertaIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) r-r.rrfr1intermediate_sizer isinstanceZ hidden_actstrr intermediate_act_fnrBrErGrHr.s   zRobertaIntermediate.__init__rsrocCs||}||}|Sr)rr)rCrsrGrGrHrVs  zRobertaIntermediate.forwardrrGrGrErHrs rcr) RobertaOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r-r.rrfrr1rr6r7r8r9r:rBrErGrHr.rzRobertaOutput.__init__rsrrocCrrrrrGrGrHrVrzRobertaOutput.forwardrrGrGrErHrrrcseZdZfddZ      ddejdeejdeejdeejd eejd eeeejd ee d eejfd dZ ddZ Z S) RobertaLayercsrt|j|_d|_t||_|j|_|j|_|jr-|js&t|dt|dd|_ t ||_ t ||_ dS)Nr z> should be used as a decoder model if cross attention is addedr&r)r-r.chunk_size_feed_forward seq_len_dimr attentionrladd_cross_attentionrbcrossattentionr intermediaterrrBrErGrHr.s   zRobertaLayer.__init__NFrsrtrurvrwrxryroc Cs|dur |ddnd}|j|||||d} | d} |jr(| dd} | d} n| dd} d} |jro|durot|dsDtd|d|durN|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nr`)ryrxrr r(rz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r|) rrlrMrbrrfeed_forward_chunkrr)rCrsrtrurvrwrxryZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputs layer_outputrGrGrHrVsP         zRobertaLayer.forwardcCs||}|||}|Sr)rr)rCrZintermediate_outputrrGrGrHr0s  zRobertaLayer.feed_forward_chunkr) rWrXrYr.r<rrrrrrVrr[rGrGrErHrs4  ArcseZdZfddZ         ddejdeejdeejd eejd eejd eeeejd ee d ee dee dee de eeje ffddZ Z S)RobertaEncodercs:t|_tfddtjD|_d|_dS)Ncsg|]}tqSrG)r).0rrDrGrH ;sz+RobertaEncoder.__init__..F) r-r.rDrZ ModuleListrangenum_hidden_layerslayergradient_checkpointingrBrErrHr.8s   zRobertaEncoder.__init__NFTrsrtrurvrwpast_key_valuesrryoutput_hidden_states return_dictroc  Cs^| rdnd} |r dnd} |r|jjrdnd} |jr%|jr%|r%tdd}|r)dnd}t|jD]^\}}| r;| |f} |durC||nd}|durM||nd}|jrc|jrc||j |||||||}n ||||||||}|d}|rz||df7}|r| |df} |jjr| |df} q0| r| |f} | st dd ||| | | fDSt ||| | | d S) NrGzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frr(r r`css|] }|dur|VqdSrrG)rvrGrGrH sz)RobertaEncoder.forward..)last_hidden_staterrs attentionscross_attentions) rDrrrrr enumeraterZ_gradient_checkpointing_func__call__tupler)rCrsrtrurvrwrrryrrZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZ layer_moduleZlayer_head_maskrxZ layer_outputsrGrGrHrV>sz         zRobertaEncoder.forward) NNNNNNFFT)rWrXrYr.r<rrrrrrrrVr[rGrGrErHr7sD      rcr) RobertaPoolercs*tt|j|j|_t|_dSr)r-r.rrfr1rZTanh activationrBrErGrHr.s zRobertaPooler.__init__rsrocCs(|dddf}||}||}|SNr)rr)rCrsZfirst_token_tensor pooled_outputrGrGrHrVs  zRobertaPooler.forwardrrGrGrErHrs rc@s,eZdZeZdZdZgdZdZddZ dS)RobertaPreTrainedModelrobertaT)r"r\rcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSt|tre|jj dSdS)zInitialize the weightsr)meanZstdNg?)rrrfweightdataZnormal_rDZinitializer_rangebiasZzero_r/r#r6Zfill_ RobertaLMHead)rCmodulerGrGrH _init_weightss       z$RobertaPreTrainedModel._init_weightsN) rWrXrYr!Z config_classZbase_model_prefixZsupports_gradient_checkpointing_no_split_modulesZ_supports_sdparrGrGrGrHrs ra The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. )Z custom_introc seZdZddgZdfdd ZddZdd Zd d Ze dd e e j de e j de e j de e j de e j de e j de e j de e j de e e j de ede ede ede edeee j effddZZS) RobertaModelr"rTcsTt|||_t||_t||_|rt|nd|_|j |_ |j |_ | dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) r-r.rDr"rUrencoderrpoolerrattn_implementationr% post_init)rCrDadd_pooling_layerrErGrHr.s    zRobertaModel.__init__cC|jjSrrUr2rCrGrGrHget_input_embeddingsz!RobertaModel.get_input_embeddingscC ||j_dSrr)rCrirGrGrHset_input_embeddings z!RobertaModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)rCZheads_to_prunerrrGrGrH _prune_headsszRobertaModel._prune_headsNrNrtr*r'rurOrvrwrrryrrroc Cs| dur| n|jj} | dur| n|jj} | dur| n|jj} |jjr-| dur(| n|jj} nd} |dur;|dur;td|durJ||||}n|durW|dd}ntd|\}}|durf|j n|j }| durv| ddj dnd}|durt |j dr|j j ddd|f}|||}|}n tj|tj|d}|j |||||d }|durtj|||f|d }|jd ko|jd ko|duo| }|r|dkr|jjrt||||}nt||j|d }n|||}|jjr'|dur'|\}}}||f}|durtj||d }|r!|dkr!t||j|d }n||}nd}|||jj}|j|||||| | | | | d }|d}|jdurO||nd}| s^||f|ddSt|||j |j!|j"|j#dS)NFzDYou cannot specify both input_ids and inputs_embeds at the same timer(z5You have to specify either input_ids or inputs_embedsrr`r*rI)rNr'r*rOrP)rJrr&)r) rtrurvrwrrryrrr )rZ pooler_outputrrsrr)$rDryruse_return_dictrlrrbZ%warn_if_padding_and_no_attention_maskr@rJrrMrUr*r>r<r?rAZonesrr%r{rrr,Zget_extended_attention_maskZinvert_attention_maskZ get_head_maskrrrrrrsrr) rCrNrtr*r'rurOrvrwrrryrrrQZ batch_sizerRrJrPrSrTZembedding_outputZuse_sdpa_attention_masksZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrZencoder_hidden_shapeZencoder_extended_attention_maskZencoder_outputssequence_outputrrGrGrHrVs         zRobertaModel.forward)T) NNNNNNNNNNNNN)rWrXrYrr.rrrrrr<rrrrrrrrVr[rGrGrErHrsb     rzS RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. c"seZdZddgZfddZddZddZe dd ee j d ee j d ee j d ee j dee j dee j dee j dee j dee j dee e e j dee dee dee dee dee e jeffddZddZZS)RobertaForCausalLMlm_head.decoder.weightlm_head.decoder.biascs@t||jstdt|dd|_t||_| dS)NzOIf you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.`Fr r-r.rlrwarningrrrlm_headrrBrErGrHr.s    zRobertaForCausalLM.__init__cCrrrdecoderrrGrGrHget_output_embeddingsrz(RobertaForCausalLM.get_output_embeddingscCrrrrCZnew_embeddingsrGrGrHset_output_embeddingsrz(RobertaForCausalLM.set_output_embeddingsNrNrtr*r'rurOrvrwlabelsrrryrrrocKs|dur|n|jj}| durd} |j||||||||| | | | |d }|d}||}d}| durE| |j} |j|| fd|jji|}|s[|f|dd}|durY|f|S|St|||j |j |j |j dS)am token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, RobertaForCausalLM, AutoConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base") >>> config = AutoConfig.from_pretrained("FacebookAI/roberta-base") >>> config.is_decoder = True >>> model = RobertaForCausalLM.from_pretrained("FacebookAI/roberta-base", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NF) rtr*r'rurOrvrwrrryrrrr0r`)losslogitsrrsrr) rDrrrrrJZ loss_functionr0rrrsrr)rCrNrtr*r'rurOrvrwr rrryrrkwargsrrprediction_scoresZlm_lossrrGrGrHrVsT2  zRobertaForCausalLM.forwardcs.d}|D]}|tfdd|Df7}q|S)NrGc3s$|] }|d|jVqdS)rN)Z index_selectrrJ)rZ past_statebeam_idxrGrHrs"z4RobertaForCausalLM._reorder_cache..)r)rCrrZreordered_pastZ layer_pastrGrrH_reorder_caches z!RobertaForCausalLM._reorder_cache)NNNNNNNNNNNNNN)rWrXrY_tied_weights_keysr.rrrrr< LongTensorrrrrrrrVrr[rGrGrErHrsh        `rcseZdZddgZfddZddZddZe dd ee j d ee j d ee j d ee j dee j dee j dee j dee j dee j dee dee dee de ee jeffddZZS)RobertaForMaskedLMrrcs@t||jrtdt|dd|_t||_| dS)NznIf you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrrrBrErGrHr. s   zRobertaForMaskedLM.__init__cCrrrrrGrGrHrrz(RobertaForMaskedLM.get_output_embeddingscCrrrrrGrGrHrrz(RobertaForMaskedLM.set_output_embeddingsNrNrtr*r'rurOrvrwr ryrrroc  Cs| dur| n|jj} |j||||||||| | | d } | d}||}d}| dur@| |j} t}||d|jj| d}| sV|f| dd}|durT|f|S|St ||| j | j dS)a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` N) rtr*r'rurOrvrwryrrrr(r`r r rsr) rDrrrrrJr rpr0rrsr)rCrNrtr*r'rurOrvrwr ryrrrrr Zmasked_lm_lossloss_fctrrGrGrHrVs<  zRobertaForMaskedLM.forward) NNNNNNNNNNNN)rWrXrYrr.rrrrr<rrrrrrrrVr[rGrGrErHrsZ      rcs0eZdZdZfddZddZddZZS)rz*Roberta Head for masked language modeling.csdtt|j|j|_tj|j|jd|_t|j|j |_ t t |j |_|j|j _dSr)r-r.rrfr1rr6r7 layer_normr0r Parameterr<r?rrBrErGrHr.es zRobertaLMHead.__init__cKs*||}t|}||}||}|Sr)rr rrrCfeaturesr rnrGrGrHrVns   zRobertaLMHead.forwardcCs,|jjjjdkr|j|j_dS|jj|_dS)Nmeta)rrrJrrrGrGrH _tie_weightsxszRobertaLMHead._tie_weights)rWrXrYrZr.rVrr[rGrGrErHrbs    rz RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZfddZe          ddeejdeejdeejdeejdeejd eejd eejd ee d ee d ee de e ej e ffddZZS) RobertaForSequenceClassificationcs>t||j|_||_t|dd|_t||_|dSNFr) r-r. num_labelsrDrrRobertaClassificationHead classifierrrBrErGrHr.s   z)RobertaForSequenceClassification.__init__NrNrtr*r'rurOr ryrrroc  Cst| dur| n|jj} |j|||||||| | d } | d} || } d}|dur|| j}|jjdurW|jdkr=d|j_n|jdkrS|jt j ksN|jt j krSd|j_nd|j_|jjdkrut }|jdkro|| | }n+|| |}n%|jjdkrt}|| d|j|d}n|jjdkrt}|| |}| s| f| d d}|dur|f|S|St|| | j| jd S) a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrtr*r'rurOryrrrr Z regressionZsingle_label_classificationZmulti_label_classificationr(r`r)rDrrr!rrJZ problem_typerr,r<rArcr squeezer rprrrsrrCrNrtr*r'rurOr ryrrrrr r rrrGrGrHrVsV     "       z(RobertaForSequenceClassification.forward NNNNNNNNNN)rWrXrYr.rrr<rrrrrrrrVr[rGrGrErHrsH      rcseZdZfddZe          ddeejdeejdeejdeejdeejd eejd eejd ee d ee d ee de e ej e ffddZZS)RobertaForMultipleChoicecs@t|t||_t|j|_t|j d|_ | dS)Nr ) r-r.rrrr8r9r:rfr1r!rrBrErGrHr.s   z!RobertaForMultipleChoice.__init__NrNr*rtr r'rurOryrrroc  Csz| dur| n|jj} |dur|jdn|jd} |dur%|d|dnd} |dur4|d|dnd} |durC|d|dnd}|durR|d|dnd}|dure|d|d|dnd}|j| | |||||| | d }|d}||}||}|d| }d}|dur||j }t }|||}| s|f|dd}|dur|f|S|St |||j |j dS)a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. Nr r(r|)r'r*rtrurOryrrr`r)rDrrrpr@rr:r!rrJr rrsr)rCrNr*rtr r'rurOryrrZ num_choicesZflat_input_idsZflat_position_idsZflat_token_type_idsZflat_attention_maskZflat_inputs_embedsrrr Zreshaped_logitsr rrrGrGrHrVsN-     z RobertaForMultipleChoice.forwardr%)rWrXrYr.rrr<rrrrrrrrVr[rGrGrErHr&sH      r&cr)RobertaForTokenClassificationcsbt||j|_t|dd|_|jdur|jn|j}t||_ t |j |j|_ | dSr)r-r.rrrclassifier_dropoutr9rr8r:rfr1r!rrCrDr(rErGrHr.Qs   z&RobertaForTokenClassification.__init__NrNrtr*r'rurOr ryrrroc  Cs| dur| n|jj} |j|||||||| | d } | d} || } || } d}|durB|| j}t}|| d|j |d}| sX| f| dd}|durV|f|S|St || | j | j dS)a- token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nr"rr(r`r) rDrrr:r!rrJr rprrrsrr$rGrGrHrV_s:   z%RobertaForTokenClassification.forwardr%)rWrXrYr.rrr<rrrrrrrrVr[rGrGrErHr'OsH     r'cs(eZdZdZfddZddZZS)r z-Head for sentence-level classification tasks.csTtt|j|j|_|jdur|jn|j}t||_ t|j|j |_ dSr) r-r.rrfr1rr(r9r8r:rout_projr)rErGrHr.s  z"RobertaClassificationHead.__init__cKsL|dddddf}||}||}t|}||}||}|Sr)r:rr<tanhr*rrGrGrHrVs     z!RobertaClassificationHead.forward)rWrXrYrZr.rVr[rGrGrErHr s  r cseZdZfddZe           ddeejdeejdeejdeejdeejd eejd eejd eejd ee d ee dee de e ej e ffddZZS)RobertaForQuestionAnsweringcs@t||j|_t|dd|_t|j|j|_| dSr) r-r.rrrrrfr1 qa_outputsrrBrErGrHr.s  z$RobertaForQuestionAnswering.__init__NrNrtr*r'rurOstart_positions end_positionsryrrroc  CsH| dur| n|jj} |j||||||| | | d } | d} || }|jddd\}}|d}|d}d}|dur|durt|dkrO|d}t|dkr\|d}|d}| d|}| d|}t |d}|||}|||}||d}| s||f| dd}|dur|f|S|St |||| j | j d S) a[ token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) Nr"rr r(rz)Z ignore_indexr`)r  start_logits end_logitsrsr)rDrrr-splitr#rrr@clampr rrsr)rCrNrtr*r'rurOr.r/ryrrrrr r0r1Z total_lossZ ignored_indexrZ start_lossZend_lossrrGrGrHrVsP          z#RobertaForQuestionAnswering.forward) NNNNNNNNNNN)rWrXrYr.rrr<rrrrrrrrVr[rGrGrErHr,sN       r,cCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor r rz)nercr<ZcumsumZtype_asrA)rNr#rPmaskZincremental_indicesrGrGrHrKs  rK)rrr&r,rr'rr)r)HrZrtypingrrrrr<Ztorch.utils.checkpoint packagingrrZtorch.nnrr r Z activationsr r Z generationrZmodeling_attn_mask_utilsrrZmodeling_outputsrrrrrrrrZmodeling_utilsrZ pytorch_utilsrrrutilsrrrZconfiguration_robertar!Z get_loggerrWrModuler"r\rrrrrrrrrrrrrrrr&r'r r,rK__all__rGrGrGrHst   (   Zf4W^ 5\^iP X