o Zh@sddlZddlmZddlmZmZmZmZddlZddl m m Z ddlm Z ddl mZmZmZddlmZddlmZdd lmZmZmZmZmZdd lmZmZdd lmZdd l m!Z!m"Z"m#Z#dd l$m%Z%ddl&m'Z'e"rddl(m)Z)ddl*m+Z+ddl,m-Z-ne.Z+e#/e0Z1Gdddej2j3Z4  dYdeej5dee6fddZ7Gddde+Z8Gddde j9Z:Gddde j9Z;Gdd d e j9Z %d[d&d'd(ej5d)ej5d*ej5d+eej?d,ee6e6fd-e6d.e6d/ee@d0eeej5ej5feej5ffd1d2ZAejBfd&d'd(ej5d3e8dej5de6d,ee6e6fd-e6d.e6d4ejCd0eej5fd5d6ZDd&d'd(ej5d)ej5d*ej5d+eej?d,ee6e6fd-e6d.e6d0eej5fd7d8ZEeDeAeEd9ZFGd:d'd'e j9ZGGd;d<dd>eZI  dYd?ej5d)ej5d+eej5d@eej5d0eej5ej5ej5e6eej5eej5ff dAdBZJd?ej5dCej5dDe6dEe6d0ej5f dFdGZKe!GdHdIdIeIZLGdJdKdKe j9ZMe!dLdMGdNdOdOeIZNe!dPdMGdQdRdReIZOe!dSdMGdTdUdUeIZPe!GdVdWdWeIZQgdXZRdS)\N) nullcontext)DictOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)_prepare_4d_attention_mask)BaseModelOutputMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)auto_docstringis_flash_attn_2_availablelogging)is_triton_available)ModernBertConfig) flash_attn_varlen_qkvpacked_func)RotaryEmbedding) apply_rotaryc@s>eZdZe  ddeejdeefddZeddZ dS) ApplyRotaryEmbUnpadN cu_seqlens max_seqlenc Csd|}|j\}}}} |ddddf|d| } t| ||d||ddd||||||_|S)NrFT)seqlen_offsetsr r! interleavedinplace) contiguousshapeviewrZsave_for_backwardr!) ctxqkvcossinr r! total_nnz_three_nheadsheaddimZqkr2a/var/www/auris/lib/python3.10/site-packages/transformers/models/modernbert/modeling_modernbert.pyforward;s   zApplyRotaryEmbUnpad.forwardc Csn|j\}}}|}|j\}}}}|ddddf|d|} t| ||d||jdddd |ddddddfS)Nr"r#rFT)r$r r!r%r& conjugate)Z saved_tensorsr'r(r)rr!) r*dor,r-r r.r/r0r1Zdqkr2r2r3backwardZs  zApplyRotaryEmbUnpad.backwardNN) __name__ __module__ __qualname__ staticmethodrtorchTensorintr4r7r2r2r2r3r:s rr r!cCst|||||S)a Arguments: qkv: (total_nnz, 3, nheads, headdim) - input tensor for packed QKV. cos, sin: (seqlen_rotary, rotary_dim / 2) interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). inplace: if True, apply rotary embedding in-place. seqlen_offsets: (batch_size,) or int. Each sequence in x is shifted by this amount. Most commonly used in inference when we have KV cache. cu_seqlens: (batch + 1,) or None max_seqlen: int Return: out: (total_nnz, dim) rotary_dim must be <= headdim Apply rotary embedding to the first rotary_dim of x. )rapply)r+r,r-r r!r2r2r3apply_rotary_unpaddedqsrAc seZdZdZ    ddededeedeejdeej f fd d Z dd ej d ej deed e ej e ej ej fffddZd efddZZS)!ModernBertUnpaddedRotaryEmbeddingzP The rotary position embeddings applied directly to unpadded sequences. @Ndimbaser!devicedtypecsTtj||d|dd||_|dur$|dur&|dur(|j|||ddSdSdSdS)a max_seqlen: if max_seqlen, device, and dtype are provided, we precompute the cos_sin_cache up to max_seqlen. If the max_seqlen, device, or dtype during training/inference differ, the cos_sin_cache will be recomputed during the forward pass. TF)rDrEZpos_idx_in_fp32rFr%NrFrG)super__init__r!_update_cos_sin_cache)selfrDrEr!rFrG __class__r2r3rJs   z*ModernBertUnpaddedRotaryEmbedding.__init__r+r returncCs6|dur|j||j|jdt||j|j||d}|S)z Apply rotary embedding *inplace* to qkv. qkv: (total_nnz, 3, nheads, headdim) cu_seqlens: (batch + 1,) cumulative sequence lengths max_seqlen: int max seq length in the batch NrHr r!)rKrFrGrAZ _cos_cachedZ _sin_cached)rLr+r r!r2r2r3r4s z)ModernBertUnpaddedRotaryEmbedding.forwardcCsd|jd|jd|jS)Nzdim=z, base=z , scale_base=)rDrEZ scale_baserLr2r2r3 extra_reprsz,ModernBertUnpaddedRotaryEmbedding.extra_repr)rCNNNN)r9r:r;__doc__r?floatrr=rFrGrJr>rrr4strrR __classcell__r2r2rMr3rBs8 rBcspeZdZdZdeffdd Zejdddejdej fd d Z dde ejd e ej dej fd dZ Z S)ModernBertEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. configcsRt||_tj|j|j|jd|_tj |j|j |j d|_ t |j|_dS)N)Z padding_idxZepsbias)rIrJrYrZ Embedding vocab_size hidden_sizeZ pad_token_idtok_embeddings LayerNormnorm_eps norm_biasnormDropoutZembedding_dropoutdroprLrYrMr2r3rJs zModernBertEmbeddings.__init__TZdynamic input_idsrOcC||||SrS)rdrbr^)rLrgr2r2r3compiled_embeddingssz(ModernBertEmbeddings.compiled_embeddingsN inputs_embedscCsH|dur|||}|S|jjr||n ||||}|SrS)rdrbrYreference_compilerir^)rLrgrj hidden_statesr2r2r3r4s zModernBertEmbeddings.forwardr8)r9r:r;rTrrJr=compile LongTensorr>rirr4rWr2r2rMr3rXs rXcs<eZdZdZdeffdd ZdejdejfddZZ S) ModernBertMLPa6Applies the GLU at the end of each ModernBERT layer. Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate` and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality. rYcsft||_tj|jt|jd|jd|_ t |j |_ t |j|_tj|j|j|jd|_dS)Nr"r[)rIrJrYrLinearr]r?Zintermediate_sizeZmlp_biasWir Zhidden_activationactrcZ mlp_dropoutrdWorerMr2r3rJs   zModernBertMLP.__init__rlrOcCs2||jddd\}}|||||S)Nr"r#rD)rrchunkrtrdrs)rLrlinputZgater2r2r3r4szModernBertMLP.forward) r9r:r;rTrrJr=r>r4rWr2r2rMr3rosroc sJeZdZd dedededeejffdd Z e e dd Z Z S) ModernBertRotaryEmbeddingNrYrDrErFcstt|dr|jdur|jd|jd|_nd|_|j|_|j|_||_ t |j|_ |j d|||d\}|_ |j d|dd|j|_dS) N rope_scaling rope_typetypedefault)rDrEinv_freqF) persistent)rIrJhasattrrygetrzmax_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenrYrZ rope_init_fnattention_scalingZregister_bufferr}Zoriginal_inv_freq)rLrYrDrErFr}rMr2r3rJs   z"ModernBertRotaryEmbedding.__init__c Cs|jddddf|jddd|j}|dddddf}t|jjtr6|jjdkr6|jjnd}t j |dd+|| dd}t j ||fdd }| |j}||j} Wdn1smwY|j|jd | j|jd fS) Nrr#rmpscpuF) device_typeenabledr"ru)rG)r}rUexpandr(torF isinstancer{rVr=Zautocast transposecatr,rr-rG) rLx position_idsZinv_freq_expandedZposition_ids_expandedrZfreqsZembr,r-r2r2r3r4s0&z!ModernBertRotaryEmbedding.forwardrS)r9r:r;rr?rUrr=rFrJno_gradrr4rWr2r2rMr3rxs &rxcCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Nr#r"ru)r(r=r)rx1Zx2r2r2r3 rotate_halfsrcCsD||}||}||t||}||t||}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. ) unsqueezer)qkr,r-rZ unsqueeze_dimZq_embedZk_embedr2r2r3apply_rotary_pos_embs  rFmoduleModernBertAttentionr+attention_masksliding_window_maskrlocal_attentionbsrDoutput_attentionsrOc Ks|j||d\} } |ddjdd\} } }t| | | | \} } |jd}t| | dd|}|dkr6|}||}tjj |dtj d  | j }tjj ||j|jd }t||}|dd}||d|}|rn||fS|fS) Nrr rr"ru࿩r#r#r#rDrG)ptraining) rotary_embrunbindrhead_dimr=matmulr functionalZsoftmaxZfloat32rrGZdropoutattention_dropoutrr'r))rr+rrrrrrDr_kwargsr,r-querykeyvaluescaleZ attn_weights attn_outputr2r2r3eager_attention_forward5s    rr target_dtypec Ks||||d}|jtjtjfv} | r1|j} ||}t||||jr$|jnd|j|d} | | } nt||||jr;|jnd|j|d} | ||fS)NrP)r r! dropout_pZ deterministicZ window_size) rGr=float16bfloat16rrrrdeterministic_flash_attnr)) rr+rr r!rrrDrrZ convert_dtypeZ orig_dtypeattnr2r2r3flash_attention_forwardZs.   rcKs|j||d\} } |ddjdd\} } } t| | | | \} } |dkr%|}tj| | | |jr0|jnd|ddd}| |d |}|fS) Nrr rr"rurr)rZ attn_maskr#) rrrrFZscaled_dot_product_attentionrrr'r))rr+rrrrrrDrr,r-rrrrr2r2r3sdpa_attention_forwards"  r)flash_attention_2eagersdpacsReZdZdZd dedeeffdd Z ddej d ee d ej fd d Z Z S)raPerforms multi-headed self attention on a batch of unpadded sequences. If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput. If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel, which requires padding and unpadding inputs, adding some overhead. See `forward` method for additional details. NrYlayer_idcs\t||_||_|j|jdkr td|jd|jd|j|_|j|_|j|_ |j|j|_ |j |j |_ t j |jd|j |jd|_||jdkr[|jd|jdf|_nd|_|j}|j}|jdkrt|jdurq|j}|j}|jd krt|j ||d |_n t||j |d |_t j |j|j|jd|_|jd krt |jnt |_t|_dS) NrzThe hidden size (z6) is not a multiple of the number of attention heads ()r rpr"rr)rDr!rE)rYrDrEr)rIrJrYrr]Znum_attention_heads ValueErrorrr num_headsr all_head_sizerrqZattention_biasWqkvZglobal_attn_every_n_layersrZglobal_rope_thetarZlocal_rope_theta_attn_implementationrBrrxrtrcIdentityout_dropsetZ pruned_heads)rLrYrZ rope_thetarrMr2r3rJs<       zModernBertAttention.__init__FrlrrOc Ks||}|jd}|jjdkr|dd|j|j}n ||dd|j|j}t|jj|f||j|j ||j |d|}|d}| | |}|f|ddS)Nrrr#r )r+rrrrDrr) rr(rYrr)rrMODERNBERT_ATTENTION_FUNCTIONrrrrrt)rLrlrkwargsr+r attn_outputsr2r2r3r4s(     zModernBertAttention.forwardrSF) r9r:r;rTrrr?rJr=r>boolr4rWr2r2rMr3rs +cseZdZddedeeffdd Zejdddej d ej fd d Z      ddej d eej deej deej deej deedee d ej fddZ ZS)ModernBertEncoderLayerNrYrcspt||_|dkrt|_n tj|j|j|j d|_t ||d|_ tj|j|j|j d|_ t ||_dS)NrrZ)rYr)rIrJrYrr attn_normr_r]r`rarrmlp_normromlp)rLrYrrMr2r3rJs  zModernBertEncoderLayer.__init__TrfrlrOcC|||SrS)rrrLrlr2r2r3 compiled_mlp z#ModernBertEncoderLayer.compiled_mlpFrrrr r!rc Csf|j||||||||d}||d}|jjr||n|||} || }|f|ddS)Nrrrr r!rrr)rrrYrkrrr) rLrlrrrr r!rrZ mlp_outputr2r2r3r4s    zModernBertEncoderLayer.forwardrS)NNNNNF)r9r:r;rrr?rJr=rmr>rrnrr4rWr2r2rMr3rs6  rc seZdZeZdZdZddgZdZdZ dZ de j fddZ e  dd ed eejd eeeeeeffd effdd ZddZfddZZS)ModernBertPreTrainedModelmodelTrXrFrcsv|jjdur ddtjdtffdd }|jj|jjtd|jj|jj|jj dd}t |t r?||j |d dSt |t rV||j|d ||j|d dSt |trm||j|d ||j|d dSt |tr|||j|d dSt |tr||j|d dSt |tttfr||j|d dSt |tjr|jjd |jdur|jjdSdSdS)Nr rstdcsRtjj|jd| ||dt|tjr%|jdur'tj|jdSdSdS)Nr)meanrab)rinitZ trunc_normal_weightrrqr[Zzeros_)rrZ cutoff_factorr2r3 init_weight<s  z.init_weightg@r)inout embedding final_outrrrrg?) rYZinitializer_cutoff_factorrModulerUZinitializer_rangemathsqrtnum_hidden_layersr]rrXr^rorrrtrrModernBertPredictionHeaddenseModernBertForMaskedLMdecoder#ModernBertForSequenceClassification ModernBertForTokenClassificationModernBertForQuestionAnswering classifierr_rdataZfill_r[Zzero_)rLrrZstdsr2rr3 _init_weights7s@        z'ModernBertPreTrainedModel._init_weightsNuse_flash_attention_2 torch_dtype device_mapcheck_device_mapc s^|jdur#d|_z |j|tj|d|dWSttfy"d|_Ynwtj||tj||dS)NrF)rrZhard_check_onlyr)rrrr)Z_attn_implementation_internalZ_check_and_enable_flash_attn_2r=rr ImportErrorrI_autoset_attn_implementation)clsrYrrrrrMr2r3rfs(  z6ModernBertPreTrainedModel._autoset_attn_implementationcCs|jjdurdSt|dr!t|jdkr!|jjrtdd|j_|jjdkr4|jjr0tdd|j_|jjdkrG|jjrCtdd|j_|jjdurTt |j_dSdS) NF hf_device_maprzqIf `accelerate` split the model across devices, `torch.compile` will not work. Falling back to non-compiled mode.rz|Compiling the model with `torch.compile` and using a `torch.mps` device is not supported. Falling back to non-compiled mode.rz|Compiling the model with `torch.compile` and using a `torch.cpu` device is not supported. Falling back to non-compiled mode.) rYrkrlenrlogger warning_oncerFr{rrQr2r2r3_maybe_set_compiles.    z,ModernBertPreTrainedModel._maybe_set_compilecs<tj|i|}|jjdvr|jjrtdd|j_|S)N>TNzcResizing token embeddings with `torch.compile` is not supported. Falling back to non-compiled mode.F)rIresize_token_embeddingsrYrkrr)rLargsrZ model_embedsrMr2r3rs z1ModernBertPreTrainedModel.resize_token_embeddings)FNNT)r9r:r;rZ config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_supports_flash_attn_2Z_supports_sdpaZ_supports_flex_attnrrr classmethodrrr=rGrrVrr?rrrrWr2r2rMr3r-s2/ rinputslabelscCs|jdtjd}tj|dd}t|}tjj tj |dtjdd}| dkr7||}n|j ^} } } | | } |j| g| R|}|durV||nd} |durb||nd}||||| |fS) a Remove padding from input sequences. Args: inputs: (batch, seqlen, ...) or (batch, seqlen) attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. position_ids: (batch, seqlen), int, position ids labels: (batch, seqlen), int, labels Returns: unpadded_inputs: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask. indices: (total_nnz) cu_seqlens: (batch + 1), the cumulative sequence lengths max_seqlen_in_batch: int unpadded_position_ids: (total_nnz) or None unpadded_labels: (total_nnz) or None r#rF)as_tupler)rrr"N)sumr=Zint32Znonzeroflattenr?maxitemrrpadZcumsumrDr(r))rrrrZseqlens_in_batchindicesZmax_seqlen_in_batchr Zunpadded_inputsbatchseqlenrestr(Zunpadded_position_idsZunpadded_labelsr2r2r3_unpad_modernbert_inputs  r rrr cCs|dkrtj|||j|jd}|||<|||}|S|j^}}tj||g|R|j|jd}|||<|j||g|R}|S)aQ Add padding to sequences. Args: inputs: (total_nnz, ...) or (total_nnz,), where total_nnz = number of tokens selected in attention_mask. indices: (total_nnz) batch: int, batch size seqlen: int, max sequence length Returns: padded_inputs: (batch, seqlen, ...) or (batch, seqlen) r)rGrF)rDr=ZzerosrGrFr)r()rrrr outputZ padded_inputs_r r2r2r3_pad_modernbert_outputs   "rc!seZdZdeffdd ZddZddZe             dd ee j d ee j d ee j d ee j d ee j dee j dee j dee dee dee dee dee dee deee j dfeffddZd e j de de j fddZZS)ModernBertModelrYcsft|_t|_tfddtjD|_ tj j j j d|_d|_|dS)Ncsg|]}t|qSr2)r).0rrYr2r3 sz,ModernBertModel.__init__..rZF)rIrJrYrX embeddingsrZ ModuleListrangerlayersr_r]r`ra final_normgradient_checkpointing post_initrerMrr3rJs   zModernBertModel.__init__cCs|jjSrSrr^rQr2r2r3get_input_embeddingssz$ModernBertModel.get_input_embeddingscCs ||j_dSrSr)rLrr2r2r3set_input_embeddings s z$ModernBertModel.set_input_embeddingsNrgrrrrjrr r! batch_sizeseq_lenroutput_hidden_states return_dictrO.c s| dur| n|jj} | dur| n|jj} | dur| n|jj} |du|duAr*td| r.dnd}| r4dnd}||durD|||durcdurc|durZ|jdd\n |jdd\|durj|jn|j}|dur|t j f|t j d}d}|jj dkrˆdur|dur|durd}|durt t||d ^}}}}Wdn1swYn#t||d ^}}}}n|durt j|d d }|j|| d \}}|j||d }|jD]?}| r||f}|jr|jr||j||||||| }n |||||||| d}|d }| r"t|dkr"||df}q| r+||f}||}|rMt|d}|durMtfdd|D}| s\tdd|||fDSt|||dS) sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers perform global attention, while the rest perform local attention. This mask is used to avoid attending to far-away tokens in the local attention layers when not using Flash Attention. indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): Indices of the non-padding tokens in the input sequence. Used for unpadding the output. cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. max_seqlen (`int`, *optional*): Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. batch_size (`int`, *optional*): Batch size of the input sequences. Used to pad the output tensors. seq_len (`int`, *optional*): Sequence length of the input sequences including padding tokens. Used to pad the output tensors. Nz:You must specify exactly one of input_ids or inputs_embedsr2r"rHFrT)rr)rFr)r)rgrjrrrrrr c3s |] }t|dVqdS)r!N)r)rhsrrrr2r3 s  z*ModernBertModel.forward..css|] }|dur|VqdSrSr2)rvr2r2r3r$s)last_hidden_staterl attentions)rYrruse_return_dictrrZ%warn_if_padding_and_no_attention_maskr(rFr=onesrrrr aranger_update_attention_maskrrrrZ_gradient_checkpointing_func__call__rrrtupler)rLrgrrrrjrr r!rrrrrZall_hidden_statesZall_self_attentionsrFZrepadr rlZ encoder_layerZ layer_outputsr2r#r3r4s!             zModernBertModel.forwardcCs|r#|jjdkrtdd|j_n|jjdkr#td|jjdt||j}t|jd d}t ||j }||jj dk d d |j}||t|jj}||fS)NrzOutputting attentions is only supported with the 'eager' attention implementation, not with "sdpa". Falling back to `attn_implementation="eager"`.rzZOutputting attentions is only supported with the eager attention implementation, not with zT. Consider setting `attn_implementation="eager"`. Setting `output_attentions=False`.r"r)rYrrrr rGr=r*r(rabsTrrrFZ masked_fillZ logical_notZfinfomin)rLrrZglobal_attention_maskrowsZdistanceZ window_maskrr2r2r3r+s&    "z&ModernBertModel._update_attention_mask NNNNNNNNNNNNN)r9r:r;rrJrrrrr=rnr>r?rrrrr4r+rWr2r2rMr3rsb       "rcs8eZdZdeffdd ZdejdejfddZZS)rrYcsNt||_t|j|j|j|_t|j |_ tj |j|j |j d|_dS)NrZ)rIrJrYrrqr]Zclassifier_biasrr Zclassifier_activationrsr_r`rarbrerMr2r3rJs  z!ModernBertPredictionHead.__init__rlrOcCrhrS)rbrsrrr2r2r3r4sz ModernBertPredictionHead.forward) r9r:r;rrJr=r>r4rWr2r2rMr3rsrzd The ModernBert Model with a decoder head on top that is used for masked language modeling. )Z custom_introc"s eZdZdgZdeffdd ZddZdejfdd Z e j d d d e j d e j fddZ e              d!dee jdee j dee j dee j dee j dee j dee j dee j deedeedeedeedeedeed eee j effdd ZZS)"rzdecoder.weightrYcs^t|||_t||_t||_tj|j |j |j d|_ |jj |_ |jj|_|dS)Nrp)rIrJrYrrrheadrrqr]r\Z decoder_biasrsparse_predictionsparse_pred_ignore_indexrrerMr2r3rJs      zModernBertForMaskedLM.__init__cCs|jSrSrrQr2r2r3get_output_embeddingssz+ModernBertForMaskedLM.get_output_embeddingsnew_embeddingscCs ||_dSrSr6)rLr8r2r2r3set_output_embeddingss z+ModernBertForMaskedLM.set_output_embeddingsTrfr rOcCrrS)rr3)rLr r2r2r3 compiled_headrz#ModernBertForMaskedLM.compiled_headNrgrrrrjrrr r!rrrrrcKsb|dur|n|jj}||jjdkr|dur|dur| dur| dur?| dur?|dur6|jdd\} } n |jdd\} } |durF|jn|j}|durXtj| | f|tjd}|durt t ||||d\}}}} }}Wdn1sywYnt ||||d\}}}} }}|j |||||||| | | | | |d }|d}|j r|dur| d}| |jdd}||jk}||}||}|jjr||n|||}d}|dur|j|||jjd }|jjdkr|jjs|durtnt t||| | d }Wdn 1swY|s'|f}|dur%|f|S|St|||j|jd S) r Nrr"rH)rrrr rgrrrrjrr r!rrrrrrr#)r\r!losslogitsrlr')rYr(rrr(rFr=r)rrr rr4r)r5rkr:rr3 loss_functionr\Zrepad_logits_with_gradrrrrlr')rLrgrrrrjrrr r!rrrrrrrFoutputsr&Z mask_tokensr>r=r r2r2r3r4s~#      zModernBertForMaskedLM.forwardNNNNNNNNNNNNNN)r9r:r;Z_tied_weights_keysrrJr7rrqr9r=rmr>r:rrrnr?rrrrr4rWr2r2rMr3rsj       rz` The ModernBert Model with a sequence classification head on top that performs pooling. c"eZdZdeffdd Ze              ddeejdeej deej deej d eej d eej d eej d eej d ee dee dee dee dee dee de e ej effddZZS)rrYcs\t||j|_||_t||_t||_tj |j |_ t |j|j|_|dSrS)rIrJ num_labelsrYrrrr3r=rrcclassifier_dropoutrdrqr]rrrerMr2r3rJWs    z,ModernBertForSequenceClassification.__init__Nrgrrrrjrrr r!rrrrrrOcKs|dur|n|jj}||j|||||||| | | | | |d }|d}|jjdkr3|dddf}n|jjdkrK||djdd|jdd d }||}||}| |}d}|dur|jj dur|j dkrpd |j_ n|j dkr|j t jks|j t jkrd |j_ nd |j_ |jj d krt}|j dkr|||}n+|||}n%|jj d krt}||d|j |d}n|jj d krt}|||}|s|f}|dur|f|S|St|||j|jdS)aB sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers perform global attention, while the rest perform local attention. This mask is used to avoid attending to far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): Indices of the non-padding tokens in the input sequence. Used for unpadding the output. cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. max_seqlen (`int`, *optional*): Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. batch_size (`int`, *optional*): Batch size of the input sequences. Used to pad the output tensors. seq_len (`int`, *optional*): Sequence length of the input sequences including padding tokens. Used to pad the output tensors. Nr;rrrr#rruT)rDZkeepdimZ regressionZsingle_label_classificationZmulti_label_classificationr<)rYr(rrZclassifier_poolingrrr3rdrZ problem_typerCrGr=longr?r squeezer r)rrrlr')rLrgrrrrjrrr r!rrrrrrr@r&Z pooled_outputr>r=loss_fctr r2r2r3r4dsn'        "       z+ModernBertForSequenceClassification.forwardrA)r9r:r;rrJrrr=rnr>r?rrrrr4rWr2r2rMr3rQs`      rzv The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks. c"rB)rrYcVt||j|_t||_t||_tj |j |_ t |j |j|_|dSrSrIrJrCrrrr3r=rrcrDrdrqr]rrrerMr2r3rJs    z)ModernBertForTokenClassification.__init__Nrgrrrrjrrr r!rrrrrrOcCs|dur|n|jj}||j|||||||| | | | | |d }|d}||}||}||}d}|durIt}||d|j |d}|s_|f|dd}|dur]|f|S|St |||j |j dS)a sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers perform global attention, while the rest perform local attention. This mask is used to avoid attending to far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): Indices of the non-padding tokens in the input sequence. Used for unpadding the output. cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. max_seqlen (`int`, *optional*): Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. batch_size (`int`, *optional*): Batch size of the input sequences. Used to pad the output tensors. seq_len (`int`, *optional*): Sequence length of the input sequences including padding tokens. Used to pad the output tensors. Nr;rr#rr<) rYr(rrr3rdrr r)rCrrlr')rLrgrrrrjrrr r!rrrrrr@r&r>r=rGr r2r2r3r4sD$   z(ModernBertForTokenClassification.forwardrA)r9r:r;rrJrrr=rnr>r?rrrrr4rWr2r2rMr3rs`      rc"seZdZdeffdd Ze             ddeejdeejdeejdeejd eejd eejd eejd eejd ee dee dee dee dee dee de e eje ffddZZS)rrYcrHrSrIrerMr2r3rJ.s    z'ModernBertForQuestionAnswering.__init__Nrgrrrstart_positions end_positionsrr r!rrrrrrOcKs|dur|n|jj}||j||||||| | | | | |d }|d}||}||}||}|jddd\}}|d }|d }d}|dur_|dur_|j ||||fi|}|sv||f|dd}|durt|f|S|St ||||j |j dS)r N) rrrrr r!rrrrrrrr#ru)r= start_logits end_logitsrlr')rYr(rrr3rdrsplitrFr'r?rrlr')rLrgrrrrJrKrr r!rrrrrrr@r&r>rLrMr=r r2r2r3r49sH#   z&ModernBertForQuestionAnswering.forwardr2)r9r:r;rrJrrr=r>r?rrrrr4rWr2r2rMr3r,s^      r)rrrrrrr8)Nrr)Sr contextlibrtypingrrrrr=Ztorch.nn.functionalrrrZtorch.nnrr r Z activationsr Zmodeling_attn_mask_utilsr Zmodeling_outputsrrrrrZmodeling_rope_utilsrrZmodeling_utilsrutilsrrrZutils.import_utilsrZconfiguration_modernbertrZflash_attn.flash_attn_interfacerZflash_attn.layers.rotaryrZflash_attn.ops.triton.rotaryrobjectZ get_loggerr9rZautogradFunctionrr>r?rArBrrXrorxrrrnrrrrGrrrrrrr rrrrrrr__all__r2r2r2r3s.          ; 5! $   .   +  $P. $ ) :  wZ [