o Zhg @s&dZddlZddlmZmZmZmZmZddlm Z ddl Z ddl m Zddl ZddlmZmZmZddlmZmZddlmZddlmZddlmZmZdd lmZd d lm Z m!Z!m"Z"m#Z#m$Z$d d l%m&Z&m'Z'm(Z(m)Z)m*Z*d d l+m,Z,m-Z-m.Z.m/Z/ddl0m1Z1e.2e3Z4dZ5dZ6ej7Z7dej8de9de9dej8fddZ:dhdej8de9de9de9dej8f ddZ;dej8de9de9dej8fddZd&ej8de9dej8fd'd(Z?d)ej8de9dej8fd*d+Z@d)ej8d,e9deej8ej8ffd-d.ZAd)ej8d,e9dej8fd/d0ZBd1ej8d2ej8d3e9dej8fd4d5ZCGd6d7d7e jDZEGd8d9d9e jDZFGd:d;d;e jDZGGdd?d?e jDZIGd@dAdAe jDZJGdBdCdCe jDZKGdDdEdEe jDZLGdFdGdGe jDZMGdHdIdIe jDZNGdJdKdKe jDZOGdLdMdMe jDZPGdNdOdOe jDZQGdPdQdQe jDZRGdRdSdSe jDZSdTZTdUZUdVZVGdWdXdXe'ZWdYZXe,dZeXGd[d\d\e jDZYGd]d^d^eWZZe(eZe5e$e6d_Z[e*eZeVe[e)eZe#e6d`e,daeXGdbdcdce jDZ\GdddedeeWZ]dfZ^e*e]eVe^e)e]e#e6d`gdgZ_dS)izFlax LongT5 model.N)AnyCallableListOptionalTuple) FrozenDictfreezeunfreeze) combine_masksmake_causal_mask) partitioning)dot_product_attention_weights) flatten_dictunflatten_dict)PRNGKey)FlaxBaseModelOutput-FlaxBaseModelOutputWithPastAndCrossAttentions%FlaxCausalLMOutputWithCrossAttentionsFlaxSeq2SeqLMOutputFlaxSeq2SeqModelOutput)ACT2FNFlaxPreTrainedModelappend_call_sample_docstring append_replace_return_docstringsoverwrite_call_docstring)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) LongT5Configzgoogle/long-t5-local-baser! input_ids pad_token_iddecoder_start_token_idreturncCsdt|}|jddddf|ddddf}|jdddf|}t|dk||}|S)z1 Shift input ids one token to the right. Nr ri)jnp zeros_likeatsetwhere)r"r#r$Zshifted_input_idsr,^/var/www/auris/lib/python3.10/site-packages/transformers/models/longt5/modeling_flax_longt5.pyshift_tokens_right:s ,r.x block_lenaxis pad_valuecCs>|j| |}dg|j}d|f||<tj||d|d}|S)zHPad an array so that a sequence length will be a multiple of `block_len`rrrconstantZ pad_widthmodeZconstant_values)shapendimr'pad)r/r0r1r2Zpad_lenr9r,r,r-_pad_to_multipleFs   r:cCsb|j||dkrt|||dd}|j||}|jd|||f|j|dd}||S)zSplit an input array into blocks of a given `block_len` along the given `axis`. If the dimension length is not a multiple of `block_len`, it will be padded first with selected `pad_value`. r)r2Nr )r7r:reshape)r/r0r1 num_blocksZ output_shaper,r,r-_split_into_blocksOs ( r= block_axis sequence_axisc Cs|j|}dg|j}d||<tj||d|d}g}tdD]}tddg|j}t|||||<t|}|||qtj||dS) zConcatenate three consecutive blocks for each input block for local attentiont. For more information, see: https://arxiv.org/pdf/2112.07916.pdf. r3r r r4r5rrNr1) r7r8r'r9rangeslicetupleappend concatenate) r/r>r?r2r<r9Z blocks_listiindicesr,r,r-_concatenate_3_blocks[s   rIcCsFtjd|tjd}||| }|dddf|dddf}|S)z:Makes 3-blocked relative position ids for local attention.rdtypeN)r'arangeint32)r0Z position_idsZcenter_position_idsrelative_position_idsr,r,r-"_make_3block_relative_position_idsqs rOlocal_attention_maskcCs:t|}t||k}|ddddddf}t||S)znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.N)rOr'abs logical_and)rPr0rNZ locality_maskr,r,r-_mask_local_attention_maskys rSattention_maskcCs^t||dd}t|ddd}|d}|ddddf}t||}t||}|ddddfS)z;Prepare attention mask to be applied for a local attention.r rAr>r?.N.N)r=rIr'rRrS)rTr0Z_blocked_attention_maskZ_3blocked_attention_maskrPr,r,r-_get_local_attention_masks  rXglobal_block_sizec s|jdd\}dtjdtjffdd }t|}tj|dd|}t|d kd d }tt||d tj d |j d }|||d}||}}|dkrjtj |j dddddf|dd}n tj |df|j d }tjt||fddd} t| |kdd} || fS)aObtain the "fixed block" global id corresponding to each input token. This implementation is a simplified version of the original Flaxformr implementation adopted from: https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py. In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for the whole fixed block, are assigned to the preceding block. Padding tokens from the original sequence are represented by -1. NrU block_idsr%csHtdk}t||dk}|dd}t||d}|S)Nr rr&rW)r'rLrRsumminimum)rZZ block_endsZtrue_block_endsZ full_blocksrYZseq_lenr,r-handle_orphan_tokenss z:_make_global_fixed_block_ids..handle_orphan_tokensr rA?g@grJrr&)Zrepeatsr1)r7npndarrayr' ones_likeZcumsumr+maximumfloorarrayrKrepeatmaxzerosones) rTrY batch_sizer^Zfixed_block_maskmaskZglobal_block_idsZ num_globalsZ_sequence_block_ids_maxglobal_segment_idsr,r]r-_make_global_fixed_block_idss" &rncCs2t||\}}|jd}t|}||d}|S)zBCreate the relative position tensor for local -> global attention.r&rW)rnr7r'rL)rTrYrZrmglobal_seq_lenZglobal_positionsside_relative_positionr,r,r- _make_side_relative_position_idss    rq hidden_statesrZrocCstj||}td||S)zFCompute individual block aggregates by summing over individual blocks.z...nd,...ng->...gd)jaxnnZone_hotr'einsum)rrrZroZone_hot_block_idsr,r,r-_create_global_aggregatessrvc@s^eZdZUeed<ejZejed<dZe ed<e j j j Zedejfed<ddZd d Zd S) FlaxLongT5LayerNorm hidden_sizerKgư>eps. weight_initcCs|d|j|jf|_dS)Nweight)paramrzrxr{selfr,r,r-setupszFlaxLongT5LayerNorm.setupcCs:t|ddjddd}|t||j}|j|S)zg Construct a layernorm module in the LongT5 style; No bias and no subtraction of mean. Zf4rUr&T)r1Zkeepdims)r'powerastypemeansqrtryr{)r~rrZvariancer,r,r-__call__s zFlaxLongT5LayerNorm.__call__N)__name__ __module__ __qualname__int__annotations__r'float32rKryfloatrsrt initializersrjrzrrarbrrr,r,r,r-rws   rwc@8eZdZUeed<ejZejed<ddZd ddZ dS) FlaxLongT5DenseActDenseconfigrKcCs|jj|jjd}|jj|jjd}tj|jjdtjj||j d|_ tj|jjdtjj||j d|_ t |jj |_t|jj|_dSNFZuse_biasZ kernel_initrK)rinitializer_factord_modeld_ffrtDensersrnormalrKwiwoDropout dropout_ratedropoutr dense_act_fnactr~Z wi_init_stdZ wo_init_stdr,r,r-rs   zFlaxLongT5DenseActDense.setupTcCs0||}||}|j||d}||}|SN deterministic)rrrr)r~rrrr,r,r-rs   z FlaxLongT5DenseActDense.__call__NT rrrr!rr'rrKrrr,r,r,r-rs rc@s6eZdZUeed<ejZejed<ddZddZ dS)FlaxLongT5DenseGatedActDenserrKcCs|jj|jjd}|jj|jjd}tj|jjdtjj||j d|_ tj|jjdtjj||j d|_ tj|jjdtjj||j d|_ t |jj|_t|jj|_dSr)rrrrrtrrsrrrKwi_0wi_1rrrrrrrrr,r,r-rs,   z"FlaxLongT5DenseGatedActDense.setupcCs>|||}||}||}|j||d}||}|Sr)rrrrr)r~rrrZ hidden_geluZ hidden_linearr,r,r-rs   z%FlaxLongT5DenseGatedActDense.__call__Nrr,r,r,r-rs  rc@r) FlaxLongT5LayerFFrrKcCs\|jjrt|j|jd|_n t|j|jd|_t|jj|jj|jd|_ t |jj |_ dS)NrJryrK)rZ is_gated_actrrKDenseReluDenserrwrlayer_norm_epsilon layer_normrtrrrr}r,r,r-r,szFlaxLongT5LayerFF.setupTcCs.||}|j||d}||j||d}|Sr)rrr)r~rrrZforwarded_statesr,r,r-r7s zFlaxLongT5LayerFF.__call__Nrrr,r,r,r-r(s  rc@seZdZUeed<dZeed<dZeed<ej Z ej ed<ddZ e dd d Z d dZddZddZejddZddZ       dddZdS)FlaxLongT5AttentionrFhas_relative_attention_biascausalrKcCs@|jj|_|jj|_|jj|_|jj|_|jj|_|jj|_ |j|j|_ |jj |j |jd}|jj |j d}|jj |j d}t j |j dtj j||jd|_t j |j dtj j||jd|_t j |j dtj j||jd|_t j |jdtj j||jd|_|jrt j|j|jtj j||jd|_dSdS)NrFrembedding_initrK)rrelative_attention_num_bucketsrelative_attention_max_distancerd_kvkey_value_proj_dim num_headsn_headsrr inner_dimrrtrrsrrrKqkvorEmbedrelative_attention_biasr~Z q_init_stdZ kv_init_stdZ o_init_stdr,r,r-rEsT            zFlaxLongT5Attention.setupT cCd}|r|d}||dk|7}t|}ntj|dd }|d}||k}|t||t||||}tj||dd}|t|||7}|dSav Adapted from Mesh Tensorflow: https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 Translate relative position to a bucket number for relative attention. The relative position is defined as memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for small absolute relative_position and larger buckets for larger absolute relative_positions. All relative positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should allow for more graceful generalization to longer sequences than the model has been trained on rrU)Za_maxr i4r'rQZcliplogr+rrelative_position bidirectional num_buckets max_distanceZrelative_bucketsZ max_exactZis_smallZrelative_position_if_larger,r,r-_relative_position_bucketss  " z-FlaxLongT5Attention._relative_position_bucketcCstj|dddddf}tj|dddddf}||}|j||j |j|jd}||}|ddddddddf}|S)%Compute binned relative position biasrrJNrrrrUrr )r'rLrrrrr transpose)r~ query_length key_lengthcontext_positionmemory_positionrrelative_position_bucketvaluesr,r,r- compute_biass "z FlaxLongT5Attention.compute_biascC ||jdd|j|jfSNrUr;r7rrr~rrr,r,r- _split_heads z FlaxLongT5Attention._split_headscCs||jdd|jfSrr;r7rrr,r,r- _merge_headssz FlaxLongT5Attention._merge_headscCs|dd}|ddtj|j|j}|ddtj|j|j}|dddd}|r||jj^} } } } |j} dt| | ddf}tj |j||}tj |j||}||_||_|jd }|j||_t t | | |kt | d || f}t||}|||fS) a\ This function takes projected key, value states from a single input token and concatenates the states to cached states from previous steps. This function is slightly adapted from the official Flax repository: https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252 cache cached_key cached_value cache_indexcSstjdtjdS)NrrJ)r'rfrMr,r,r,r-sz;FlaxLongT5Attention._concatenate_to_cache..rrr ) has_variablevariabler'rir7rKvaluelenrslaxdynamic_update_slice broadcast_torLrDr )r~keyrqueryrTZis_initializedrrrZ batch_dims max_lengthrZdepth_per_headZ cur_indexrHZnum_updated_cache_vectorsZpad_maskr,r,r-_concatenate_to_caches(     z)FlaxLongT5Attention._concatenate_to_cachec Cs|jo |ddo | }|jd}|r|n|jd} |jr$|| |} n|dur.t|} n tjd|j| |f|j d} |rX|j ddjd} t j | dd|dfd|j|| f} | S)Nrrr rJr)rrr7rrr'r(rirrK variablesrsr dynamic_slice) r~ key_states query_statesrT init_cache seq_lengthcausal_attention_mask_shiftZcache_is_filledrr position_biasmax_decoder_lengthr,r,r-_create_position_biass     z)FlaxLongT5Attention._create_position_biasNc  Csr|jdd\} } ||} |dur||n||} |dur%||n||} || } || } || } | t| jd9} |ddrS|jrS|j ddnd}|jrt |dd }|ddr~|j ddjd }t j |dd|dfd d | |f}t|| f|jd d}ttj|d d |j}t||}n |durtj|d d }|jr|dds|r|| | | |\} } }|durt|jj}t j |dkt|jd |jt|j||j}|dur|| | ||| |}|dur||}d}|s |jd kr |d}t| | |||jd||jd}td|| }||}||}||f}|r7||f}|S) Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). NrUr&rrrrboolrJr )rAr_rTZbias dropout_rngrZbroadcast_dropoutrrK...hqk,...khd->...qhd)r7rrrrr'rrrrr rsrrrZ expand_dimsr rZfinforKminselectfullrrrmake_rngr rurr)r~rrrTkey_value_statesr use_cacheoutput_attentionsrrrkrrr value_statesrZcausal_attention_maskrZ mask_valuer attn_weights attn_outputoutputsr,r,r-rs    "           zFlaxLongT5Attention.__call__Trr)NNNFFTF)rrrr!rrrrr'rrKr staticmethodrrrrrtcompactrrrr,r,r,r-r?s,   . #  rc@seZdZUeed<dZeed<ejZ ej ed<ddZ e dd d Z d e fd dZddZddZde deejdejfddZ     dddZdS)FlaxLongT5LocalAttentionrFrrKcCsR|jj|_|jj|_|jj|_|jj|_|jj|_|jj|_|jd|_ |jj |_ |j|j|_ |jj |j |jd}|jj |j d}|jj |j d}tj|j dtjj||jd|_tj|j dtjj||jd|_tj|j dtjj||jd|_tj|jdtjj||jd|_|jrtj|j|jtjj|d|_dSdS)Nr rFrr)rrrrrrrr local_radiusr0rrrrrtrrsrrrKrrrrrrrrr,r,r-r`sV              zFlaxLongT5LocalAttention.setupTrrcCrrrrr,r,r-r " z2FlaxLongT5LocalAttention._relative_position_bucket block_lengthcCtjd|dd}||| }|dddf|dddf}|j|d|j|jd}||}|dddddddddf}|SrrrrJNTrrr'rLrrrrrr~rrrrrrr,r,r-r  $z%FlaxLongT5LocalAttention.compute_biascCrrrrr,r,r-rrz%FlaxLongT5LocalAttention._split_headscC||jdd|jSNrr&rrr,r,r-rz%FlaxLongT5LocalAttention._merge_headsr0rTr%cCN|jr ||}|S|durt|}|Stjdd|j|d|f|jd}|SNr rrJrrr'r(rirrKr~r0rTrr,r,r-r   z.FlaxLongT5LocalAttention._create_position_biasNc Cs|jdd\}}||} |dur||n||} |dur%||n||} || } || } || } t| |jdd} t| |jdd} t| |jdd} t| ddd} t| ddd} | t | jd9} |durt ||j}t j |dkt|jd|jt|jd |j}|dur||j|}|dur||dd}d} |s|jdkr|d } t| | || |jd ||jd } td | | }||}|ddd|ddf}||}||f}|r|| f}|S)rNrUr rArVr&rr_ _rTrr)r7rrrrr=r0rIr'rrXrsrrrrrKrswapaxesrrr rurr)r~rrrTrrrrrkrrrr rr r r r,r,r-rsZ         z!FlaxLongT5LocalAttention.__call__r NNNFT)rrrr!rrrr'rrKrrrrrrrrrarbrrr,r,r,r-r[s"  / #rc@seZdZUeed<dZeed<ejZ ej ed<ddZ e dd d Z d e fd dZdejdejdejfddZddZddZde deejdejfddZ     dddZdS) "FlaxLongT5TransientGlobalAttentionrFrrKcCs|jj|_|jj|_|jj|_|jj|_|jj|_|jj|_|jd|_ |jj |_ |jj |_ |j|j|_ |jj|j |jd}|jj|j d}|jj|j d}tj|j dtjj||jd|_tj|j dtjj||jd|_tj|j dtjj||jd|_tj|jdtjj||jd|_|jrtj|j|jtjj|d|_|jrtj|j|jtjj|d|_t|jj|jj|jd|_dS)Nr rFrrr) rrrrrrrrrr0rYrrrrrtrrsrrrKrrrrrrrglobal_relative_attention_biasrwrglobal_input_layer_normrr,r,r-r3sh                z(FlaxLongT5TransientGlobalAttention.setupTrrcCrrrrr,r,r-rnrzs  r>c@s@eZdZUeed<ejZejed<ddZ    d dd Z dS) FlaxLongT5LayerCrossAttentionrrKcCsDt|jdd|jd|_t|jj|jj|jd|_t |jj |_ dS)NFr?r) rrrKEncDecAttentionrwrrrrtrrrr}r,r,r-rr0z#FlaxLongT5LayerCrossAttention.setupNFTc CsJ||}|j|||||d}||j|d|d}|f|dd} | S)N)rTrrrrrr )rrDr) r~rrrrTrrrr5r6r r,r,r-rr7z&FlaxLongT5LayerCrossAttention.__call__r8rr,r,r,r-rCs  rCc@sVeZdZUeed<dZeed<ejZ ej ed<ddZ         d d d Z dS) FlaxLongT5BlockrFrrKcCs|jj|_|jr t}n|jjdkrt}n|jjdkrt}n td|jjd||j|jtd|j df|_ d}|jrP|j t |jtd|j df7_ |d7}|j t |jt||j df7_ dS) Nlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got .r)rnamerKr )rHrK) rrr>Zencoder_attention_typer+r< ValueErrorrstrrKlayerrCr)r~Zattention_layerZfeed_forward_indexr,r,r-rs0   "&zFlaxLongT5Block.setupNTc Cs|jd||||| | d} | d}| dd} |jo|du} | r9|jd|||||| d}|d}| |dd} |jd|| d}|f}|| }|S)NrrAr )rrTrrrr&r)rKr)r~rrrTrencoder_hidden_statesencoder_attention_maskencoder_decoder_position_biasr return_dictrrZself_attention_outputsZattention_outputsZdo_cross_attentionZcross_attention_outputsr r,r,r-r s4  zFlaxLongT5Block.__call__) NNNNNFTTFrBr,r,r,r-rEs   rEc@sPeZdZUeed<eed<ejZejed<ddZ        d d d Z dS) FlaxLongT5LayerCollectionrrrKcCst|j|j|jd|_dS)Nr.)rErrrKrKr}r,r,r-rBs  zFlaxLongT5LayerCollection.setupNFTc Cs|j||||||||| d S)N)rTrrLrMrNrrr)rK) r~rrrTrrLrMrNrrrr,r,r-rGs z"FlaxLongT5LayerCollection.__call__)NNNNNFTF) rrrr!rrr'rrKrrr,r,r,r-rP=s rPc @sfeZdZUeed<ejZejed<dZe ed<ddZ        dd e d e d e d e fd dZ dS)FlaxLongT5BlockCollectionrrKFgradient_checkpointingcs^jj_jrttddfddtjjD_dSfddtjjD_dS)N))Zstatic_argnumscs(g|]}j|dkjt|dqSr)rrKrH)rrKrJ.0rGZFlaxLongT5CheckpointLayerr~r,r- jz3FlaxLongT5BlockCollection.setup..cs(g|]}tj|dkjt|dqSrV)rPrrKrJrWr}r,r-rZtr[)rrrRrematrPrB num_layersblocksr}r,rYr-rfs      zFlaxLongT5BlockCollection.setupNTroutput_hidden_statesrrc  Cs|rdnd} |r dnd} |r|jrdnd} d} d} t|jD]A\}}|r)| |f} |||| ||| ||| }|d}|d} |jrL|durL||rIdnd} |r_| |df} |jr_| |df} qt|| | | dS)Nr,rr rrUlast_hidden_staterr attentionscross_attentions)r enumerater^r)r~rrrTrLrMrr_rrall_hidden_statesZall_attentionsZall_cross_attentionsrrNrGZ layer_moduleZ layer_outputsr,r,r-r~sD   z"FlaxLongT5BlockCollection.__call__)NNNNFFTF) rrrr!rr'rrKrRrrrr,r,r,r-rQas,   rQc @sveZdZUeed<ejed<ejZ ej ed<dZ e ed<ddZ       dd e d e d e d e de f ddZ dS)FlaxLongT5Stackr embed_tokensrKFrRcCsN|jj|_t|j|j|jd|_t|jj|jj|jd|_ t |jj |_ dS)NrKrRr)rrrQrKrRblockrwrrfinal_layer_normrtrrrr}r,r,r-rs  zFlaxLongT5Stack.setupNTrr_rOrrc Cs||} |j| |d} |j| ||||||| d} | d} || } |j| |d} d} |r5| j} | | f} |sL|rC| | f| ddS| f| ddSt| | | j| jdS)Nr)rTrLrMrr_rrrrUr ra)rhrrjrkrrrrcrd) r~r"rTrLrMrr_rOrrrrr rfr,r,r-rsB    zFlaxLongT5Stack.__call__) NNNNFFTTF)rrrr!rrtrr'rrKrRrrrr,r,r,r-rgs4      rga Args: input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 Training](./longt5#training). attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a1 Args: decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) For training, `decoder_input_ids` should be provided. encoder_outputs (`tuple(tuple(jnp.ndarray)`): Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`): Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a Args: input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 Training](./longt5#training). attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5 Training](./longt5#training). decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. encoder_outputs (`tuple(tuple(jnp.ndarray)`, *optional*): Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(jnp.ndarray))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cseZdZUdZeZdZdZej e d<dde j dfded e ed ed e jd ef fd d ZddZd+dejjd e dedefddZee         d,de jdee jde jdee jdeedeedeededeedefd d!Zd"d#Zee e!e"ed$       d-de jdee jdeedeedeededeedefd%d&Z#ee$e!e%ed$         d,d'ee jdee jd(eedeedeedeededeedefd)d*Z&Z'S).FlaxLongT5PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. Z transformerN module_classr@rTr input_shapeseedrK_do_initc s2|jd||d|}tj||||||ddS)N)rrK)rnrorKrpr,)rmsuper__init__)r~rrnrorKrpr1module __class__r,r-rrs z"FlaxLongT5PreTrainedModel.__init__cCs|j|j|jdd|_dS)NT)rrKrR)rmrrK_moduler}r,r,r-enable_gradient_checkpointings  z7FlaxLongT5PreTrainedModel.enable_gradient_checkpointingrngparamsr%c Cstj|dd}t|}t|}t|}tj|\}} || d} |j| ||||d} |durUtt | } tt |}|j D]} | | || <qBt |_ t t |S| S)NrrJ)ryrry)r'rircrsrandomsplitrsinitrr Z _missing_keysr*rr) r~rxrnryr"rTdecoder_input_idsdecoder_attention_maskZ params_rngrrngsZ random_paramsZ missing_keyr,r,r- init_weightss.        z&FlaxLongT5PreTrainedModel.init_weightsFr"rTr}r~rr_rOtrainrc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur&td|dur/t|}|dur8t|}| dur@d| ini} |jjd| pJ|j itj |ddtj |ddtj |ddtj |dd|||| | d S)NzfMake sure to provide both `input_ids` and `decoder_input_ids`. `decoder_input_ids` is not passed here.rryrrJ) r"rTr}r~rr_rOrr) rrr_rOrIr'rcrsapplyryrf) r~r"rTr}r~rr_rOrryrrr,r,r-rs2       z"FlaxLongT5PreTrainedModel.__call__cCsTtj||fdd}t|}dd}|jjtjd|||dd|d}t|dS) a+ Args: batch_size (`int`): batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache. max_length (`int`): maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized cache. encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`): `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. rrJc[|}|||fi|SN_get_decoder_modulersr}r~r1decoder_moduler,r,r-_decoder_forwardz>FlaxLongT5PreTrainedModel.init_cache.._decoder_forwardrT)r}r~rLrmethodr) r'rjrcrsr|rsrzrr )r~rkrencoder_outputsr}r~rZinit_variablesr,r,r-rs   z$FlaxLongT5PreTrainedModel.init_cache output_type config_classc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur't|}i} |dur1|| d<dd} |jjd|p=|jitj |ddtj |dd|||| | | d S) a Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) ```Nrc[rr)_get_encoder_module)rsr"rTr1Z encode_moduler,r,r-_encoder_forward;sz:FlaxLongT5PreTrainedModel.encode.._encoder_forwardryrrJ)r"rTrr_rOrrr) rrr_rOr'rcrsrryrf) r~r"rTrr_rOrryrrrr,r,r-encodes*    z FlaxLongT5PreTrainedModel.encoderMpast_key_valuesc Csx|dur|n|jj}|dur|n|jj}|dur|n|jj}|d} |dur6| jdd\} }t| |f}|j\} }|durFt| |f}i}| durP| |d<d| pU|ji}|ra||d<dg}nd}dd }|jj |tj |d d tj |d d | tj |d d |||| |||d }|dur|r|\}}t |d|d <|S|dur|s|\}}|ddt |df|dd}|S)aG Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> import jax.numpy as jnp >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) >>> decoder_start_token_id = model.config.decoder_start_token_id >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id >>> outputs = model.decode(decoder_input_ids, encoder_outputs) >>> logits = outputs.logits ```NrrUrryrFc[rrrrr,r,r-rrz:FlaxLongT5PreTrainedModel.decode.._decoder_forwardrrJ r}r~rLrMrr_rOrrmutablerrr ) rrr_rOr7r'rjryrsrrfr )r~r}rrMr~rrr_rOrryrrLrksequence_lengthrinputsrrr pastr,r,r-decodeKsT&      &z FlaxLongT5PreTrainedModel.decoder NNNNNNFNN)NNNNFNN)(rrrr:r!rZbase_model_prefixrmrtModulerr'rrrrKrrrrwrsrzrrrrLONGT5_INPUTS_DOCSTRINGrbrdictrrrLONGT5_ENCODE_INPUTS_DOCSTRINGrrrLONGT5_DECODE_INPUTS_DOCSTRINGrr __classcell__r,r,rtr-rls     1$  8     rla The LongT5 model was proposed in [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung and Yinfei Yang. It's an encoder-decoder transformer pre-trained in a text-to-text denoising generative setting. LongT5 model is an extension of T5 model, and it enables using one of the two different efficient attention mechanisms - (1) Local attention, or (2) Transient-Global attention. This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior. Finally, this model supports inherent JAX features such as: - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) Parameters: config ([`LongT5Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights. dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and `jax.numpy.bfloat16` (on TPUs). This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If specified all the computation will be performed with the given `dtype`. **Note that this only specifies the dtype of the computation and does not influence the dtype of model parameters.** If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`]. z_The bare LONGT5 Model transformer outputting raw hidden-stateswithout any specific head on top.c@leZdZUeed<ejZejed<dZe ed<ddZ ddZ d d Z dd e fddZ d S)FlaxLongT5ModulerrKFrRcC|jSrencoderr}r,r,r-rz$FlaxLongT5Module._get_encoder_modulecCrrdecoderr}r,r,r-rrz$FlaxLongT5Module._get_decoder_modulecCstj|jj|jjtjj|jjd|j d|_ t |j}d|_ t||j |j |jd|_t |j}d|_ |jj|_t||j |j |jd|_dS)Nr`rF)rhrKrRT)rtrr vocab_sizerrsrrrrKsharedcopydeepcopyrrgrRrnum_decoder_layersr]rr~Zencoder_configZdecoder_configr,r,r-rs.    zFlaxLongT5Module.setupNTrc Cs||dur|n|jj}|j|||||| d}|j|||d||||| d} |s*| |St| j| j| j| j| j |j|j|jdS)Nr"rTrr_rOrrr"rTrLrMrr_rOr)rbrdecoder_hidden_statesdecoder_attentionsrdencoder_last_hidden_staterLencoder_attentions) ruse_return_dictrrrrbrrrrcrd) r~r"rTr}r~rrr_rOrdecoder_outputsr,r,r-rs>   zFlaxLongT5Module.__call__ NNNNNNNNTrrrr!rr'rrKrRrrrrrr,r,r,r-rs&   rc@seZdZeZdS)FlaxLongT5ModelN)rrrrrmr,r,r,r-r;sra Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxLongT5Model >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") >>> model = FlaxLongT5Model.from_pretrained("google/long-t5-local-base") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="np" ... ).input_ids >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="np").input_ids >>> # forward pass >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) >>> last_hidden_states = outputs.last_hidden_state ``` rz4LONGT5 Model with a `language modeling` head on top.c@r)(FlaxLongT5ForConditionalGenerationModulerrKFrRcCrrrr}r,r,r-rcrz>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> import jax.numpy as jnp >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "summarize: My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) >>> decoder_start_token_id = model.config.decoder_start_token_id >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id >>> outputs = model.decode(decoder_input_ids, encoder_outputs) >>> logits = outputs.logits ```NrrUrryrFc s|}|||fi|}|d}jjr|jjd}jjr9|jjdd}|jdd|jii|}||fS||}||fS)Nrrryrr) rrrrrrrrr) rsr}r~r1rrrrrr}r,r-r s  zCFlaxLongT5ForConditionalGeneration.decode.._decoder_forwardrrJr)rrrrcrdr r)rrr_rOr7r'rjryrsrrfrrrrcrdr )r~r}rrMr~rrr_rOrryrrLrkrrrrrr rrrr,r}r-rsf&         &z)FlaxLongT5ForConditionalGeneration.decoderTc KsP|j\}}||||} tj||fdd} |dur!tj| |d} | ||| dS)NrrJr3)rrrMr~)r7rr'rjrsrr) r~r}rrTr~rr1rkrrZextended_attention_maskr,r,r-prepare_inputs_for_generationR s z@FlaxLongT5ForConditionalGeneration.prepare_inputs_for_generationcCs|j|d<|S)Nr)r)r~Z model_outputsZ model_kwargsr,r,r-update_inputs_for_generationo s z?FlaxLongT5ForConditionalGeneration.update_inputs_for_generationr)NNN)rrrrrmrrrrr!rr'rbrrrrrsZArrayrrr,r,r,r-rsT      ra Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> ARTICLE_TO_SUMMARIZE = "summarize: My friends are cool but they eat too many carbs." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], return_tensors="np") >>> # Generate Summary >>> summary_ids = model.generate(inputs["input_ids"]).sequences >>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)) ``` )rrrlr)`r:rtypingrrrrrZ flax.linenZlinenrtrsZ jax.numpynumpyr'raZflax.core.frozen_dictrrr r r r Znn_partitioningZflax.linen.attentionr Zflax.traverse_utilrrZ jax.randomrZmodeling_flax_outputsrrrrrZmodeling_flax_utilsrrrrrutilsrrrrZconfiguration_longt5r!Z get_loggerrloggerZ_CHECKPOINT_FOR_DOCZ_CONFIG_FOR_DOCr\rbrr.r:r=rIrOrSrXrnrqrvrrwrrrrrr%r+r<r>rCrErPrQrgrrrrlZLONGT5_START_DOCSTRINGrrZFLAX_LONGT5_MODEL_DOCSTRINGrrZ,FLAX_LONGT5_CONDITIONAL_GENERATION_DOCSTRING__all__r,r,r,r-s       $  $ $*  'T"&')#V$WG*<4)Zn)