o
    Zh
                    @   sL  d dl mZ d dlmZmZmZmZmZ ddlm	Z	 ddl
mZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ e* rd dl0Z0d dl0m1Z1 e' rd dl2m3Z3 ddl4m5Z5 e(6e7Z8edG dd de1j9Z:G dd de1j9Z;G dd de1j9Z<G d d! d!e1j9Z=G d"d# d#e1j9Z>d$d% Z?G d&d' d'e1j9Z@G d(d) d)e1j9ZAG d*d+ d+e1j9ZBd,d- ZCd[d.d/ZDd0e0jEd1eFd2e0jEfd3d4ZG	5d\d6e1j9d7e0jEd8e0jEd9e0jEd:ee0jE d;eHd<eHfd=d>ZIG d?d@ d@e1j9ZJG dAdB dBeZKe%G dCdD dDe ZLe%G dEdF dFe ZMG dGdH dHe1j9ZNe%G dIdJ dJeLZOG dKdL dLee$ZPe%G dMdN dNeLeZQeG dOdP dPeZReG dQdR dReZSe%dSdTG dUdV dVeMZTe%dWdTG dXdY dYeMeZUg dZZVdS )]    )	dataclass)CallableListOptionalTupleUnion   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPastModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)
LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging)is_torch_available   )	AutoModel   )
AriaConfigAriaTextConfigN)nn)	BlockMask)make_flex_block_causal_maskZRMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	AriaTextRMSNormư>c                    s&   t    tt|| _|| _dS )z>
        AriaTextRMSNorm is equivalent to T5LayerNorm
        N)super__init__r$   	ParametertorchZonesweightvariance_epsilon)selfhidden_sizeeps	__class__ U/var/www/auris/lib/python3.10/site-packages/transformers/models/aria/modeling_aria.pyr*   ?   s   

zAriaTextRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )Nr   T)Zkeepdim)	dtypetor,   float32powmeanZrsqrtr.   r-   )r/   hidden_statesZinput_dtypeZvariancer4   r4   r5   forwardG   s
   zAriaTextRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler-   shaper.   r/   r4   r4   r5   
extra_reprN   s   zAriaTextRMSNorm.extra_repr)r(   )__name__
__module____qualname__r*   r=   rA   __classcell__r4   r4   r2   r5   r'   =   s    r'   c                       (   e Zd ZdZ fddZdd Z  ZS )AriaProjectorMLPa!  
    Feed-Forward Network module for the Aria Projector.

    Args:
        in_features (`int`):
            Input embedding dimension.
        hidden_features (`int`):
            Hidden dimension of the feed-forward network.
        output_dim (`int`):
            Output dimension.
    c                    s<   t    tj||dd| _tj||dd| _td | _d S )NFbiasZgelu_new)r)   r*   r$   Linear	linear_in
linear_outr	   act)r/   in_featureshidden_features
output_dimr2   r4   r5   r*   _   s   
zAriaProjectorMLP.__init__c                 C   s   |  | |}| |}|S N)rM   rK   rL   )r/   r<   r4   r4   r5   r=   e   s   
zAriaProjectorMLP.forwardrB   rC   rD   __doc__r*   r=   rE   r4   r4   r2   r5   rG   R   s    rG   c                       s6   e Zd ZdZd
dedef fddZddd	Z  ZS )AriaCrossAttentionzv
    Aria Cross-Attention module.

    Args:
        config (`AriaConfig`):
            The configuration to use.
    r   configdropout_ratec                    s   t    |jj}|jj}|| _tj||dd| _tj||dd| _	tj||dd| _
tj||dd| _t||| _t|| _t|| _t|| _d S )NFrH   T)Zbatch_first)r)   r*   vision_configr0   num_attention_heads	num_headsr$   rJ   q_projk_projv_projMultiheadAttentionmultihead_attnlinearZDropoutdropout	LayerNorm
layer_normlayer_norm_kv)r/   rU   rV   r0   rY   r2   r4   r5   r*   t   s   
zAriaCrossAttention.__init__Nc           	      C   sX   |  | |}| |}| |}| |}| j||||d\}}| | |}|S )a  
        Forward pass of the AriaCrossAttention module.

        Args:
            key_value_states (`torch.Tensor`):
                Input tensor for key and value.
            hidden_states (`torch.Tensor`):
                Input tensor for query.
            attn_mask (`torch.Tensor`, *optional*, defaults to None):
                Attention mask.

        Returns:
            torch.Tensor:
                Output tensor after cross-attention.
        	attn_mask)rZ   rb   rc   r[   r\   r^   r`   r_   )	r/   key_value_statesr<   re   querykeyvalueattn_output_r4   r4   r5   r=      s   


zAriaCrossAttention.forward)r   rQ   )	rB   rC   rD   rS   r"   floatr*   r=   rE   r4   r4   r2   r5   rT   k   s    rT   c                       sB   e Zd ZdZdef fddZd
dejdeej fdd	Z	  Z
S )AriaProjectora  
    Aria Projector module.

    This module projects vision features into the language model's embedding space, enabling interaction between vision and language components.

    Args:
        config (`AriaConfig`):
            Configuration object for the model.
    rU   c                    s   t    |j| _|jj| _|jj| _|jj| _	|j
j| _|j
j| _tt|j| j| _t|| _t| j| _t| j| j| j| _d S rQ   )r)   r*   Zprojector_patch_to_query_dictpatch_to_query_dictrW   r0   rN   rX   rY   Zkv_dimtext_configrO   rP   r$   r+   r,   zerosZ'max_value_projector_patch_to_query_dictrg   rT   
cross_attnra   rb   rG   feed_forwardr/   rU   r2   r4   r5   r*      s   






zAriaProjector.__init__Nrf   re   c           	      C   s   |j d |j d }}|| j vr td| d| j  d| j| }| jd| d|dd}|durL|| jd}|d	d|
dd}| j|||d}| | |}|S )	a  
        Forward pass of the Projector module.

        Args:
            key_value_states (`torch.Tensor`):
                Input tensor of shape (batch_size, num_patches, kv_dim).
            attn_mask (`torch.Tensor`, *optional*, default is None):
                Attention mask.

        Returns:
            `torch.Tensor`: Output tensor of shape (batch_size, query_number, output_dim).
        r   r!   zNumber of patches z: not found in patch_to_query_dict amongst possible values .Nr6   rd   )r?   rn   keysKeyErrorrg   	unsqueezerepeatZrepeat_interleaverY   expandsizerq   rr   rb   )	r/   rf   re   
batch_sizeZnum_patchesZ	query_numZqueriesZattention_outoutr4   r4   r5   r=      s   
zAriaProjector.forwardrQ   )rB   rC   rD   rS   r"   r*   r,   Tensorr   r=   rE   r4   r4   r2   r5   rm      s    
$rm   c                       s.   e Zd ZdZdef fddZdd Z  ZS )AriaSharedExpertsMLPa/  
    Shared Expert MLP for shared experts.

    Unlike routed experts, shared experts process all tokens without routing.
    This class reconfigures the intermediate size in comparison to the LlamaMLP.

    Args:
        config (`AriaTextConfig`): Configuration object for the Aria language model.
    rU   c                    s~   t    || _|j| _|j|j | _tj| j| j|jd| _	tj| j| j|jd| _
tj| j| j|jd| _t|j | _d S )NrH   )r)   r*   rU   r0   intermediate_sizeZmoe_num_shared_expertsr$   rJ   Zmlp_bias	gate_projup_proj	down_projr	   Z
hidden_actact_fnrs   r2   r4   r5   r*      s   
zAriaSharedExpertsMLP.__init__c                 C   s$   |  | | || | }|S rQ   )r   r   r   r   )r/   xr   r4   r4   r5   r=      s    zAriaSharedExpertsMLP.forwardrB   rC   rD   rS   r#   r*   r=   rE   r4   r4   r2   r5   r~      s    

r~   c                 C   s   | j d }|j d }tj||| j| jd}tj|dd}tjdtj|jd}t||f}t|j d D ] }|| }	||d  }
| |	|
 }t	||| }|||	|
< q4|S )a*  
    Compute the matrix multiplication (GEMM) for each expert sequentially. This approach is computationally inefficient, especially when dealing with a large number of experts.

    Args:
        token_states (torch.Tensor): Input tensor of shape (num_tokens, in_features).
        expert_weights (torch.Tensor): Weight tensor of shape (num_experts, in_features, out_features).
        tokens_per_expert (torch.Tensor): Number of tokens assigned to each expert.

    Returns:
        torch.Tensor: Output tensor of shape (num_tokens, out_features).
    r   r6   r7   devicedimr!   )
r?   r,   rp   r7   r   Zcumsumlongcatrangematmul)Ztoken_statesZexpert_weightstokens_per_expertZ
num_tokensout_featuresoutputZcumsum_num_tokensZzero_tensorZ
expert_numstartendtokensr|   r4   r4   r5   sequential_experts_gemm   s   

r   c                       rF   )AriaGroupedExpertsGemmaP  
    Grouped GEMM (General Matrix Multiplication) module for efficient expert computation.
    This module utilizes the grouped_gemm library (https://github.com/fanshiqing/grouped_gemm)
    for optimized performance. If the grouped_gemm library is not installed, it gracefully
    falls back to a sequential GEMM implementation, which may be slower but ensures
    functionality.

    Args:
        in_features (`int`):
            Number of input features.
        out_features (`int`):
            Number of output features.
        groups (`int`):
            Number of expert groups.
    c                    s6   t    || _|| _|| _tt|||| _	d S rQ   )
r)   r*   rN   r   groupsr$   r+   r,   emptyr-   )r/   rN   r   r   r2   r4   r5   r*   -  s
   
zAriaGroupedExpertsGemm.__init__c                 C   s   t || j| S )au  
        Perform grouped matrix multiplication.

        Args:
            input (`torch.Tensor`):
                Input tensor of shape (num_tokens, in_features).
            tokens_per_expert (`torch.Tensor`):
                Number of tokens assigned to each expert.

        Returns:
            torch.Tensor: Output tensor of shape (num_tokens, out_features).
        )r   r-   cpu)r/   inputr   r4   r4   r5   r=   4  s
   zAriaGroupedExpertsGemm.forwardrR   r4   r4   r2   r5   r     s    r   c                       s2   e Zd ZdZdeddf fddZdd Z  ZS )	AriaGroupedExpertsMLPz
    Grouped MLP module for Mixture of Experts.

    Args:
        config (`AriaTextConfig`):
            Configuration object for the model.
    rU   returnNc                    s@   t    || _t|j|jd |j| _t|j|j|j| _d S )Nr   )	r)   r*   rU   r   r0   r   moe_num_expertsfc1fc2rs   r2   r4   r5   r*   Q  s   
zAriaGroupedExpertsMLP.__init__c                 C   s@   |  ||}tj|ddd\}}tj|| }| ||}|S )a5  
        Forward pass of the Grouped MLP.

        Args:
            permuted_tokens (torch.Tensor): Permuted input tokens.
            tokens_per_expert (torch.Tensor): Number of tokens assigned to each expert.

        Returns:
            torch.Tensor: Output tensor after passing through the MLP.
        r   r6   r   )r   r,   chunkr$   
functionalZsilur   )r/   permuted_tokensr   Z
fc1_outputZ
projectionZgateZ
fc2_outputr4   r4   r5   r=   W  s
   zAriaGroupedExpertsMLP.forwardr   r4   r4   r2   r5   r   H  s    r   c                       s<   e Zd ZdZdef fddZdejdejfddZ  Z	S )	AriaTextMoELayerz
    Aria Text Mixture of Experts (MoE) Layer.

    This layer applies a gating mechanism to route input tokens to different experts.

    Args:
        config (`AriaTextConfig`):
            Configuration object for the text component of the model.
    rU   c                    s>   t    tj|j|jdd| _t|| _t	|| _
|| _d S NFrH   )r)   r*   r$   rJ   r0   r   routerr   expertsr~   shared_expertsrU   rs   r2   r4   r5   r*   u  s
   



zAriaTextMoELayer.__init__r<   r   c                 C   s>  |j }|d|d}| |}tj|| jjdd\}}tj	j
|dd}|j}tj| tj| jjd| jjd d|}|}	|	d}
t|
}|d|| jj }| ||}tj|j d | jj |df|j|jd}|d|| |d| jj|d}||d jdd|}| ||}|| S )a.  
        Forward pass of the MoE Layer.

        Args:
            hidden_states (`torch.Tensor`):
                Input tensor of shape (batch_size, sequence_length, hidden_size).

        Returns:
            torch.Tensor: Output tensor after passing through the MoE layer.

        Process:
        1. Route tokens to experts using the router.
        2. Permute tokens based on routing decisions.
        3. Process tokens through experts.
        4. Unpermute and combine expert outputs.
        5. Add shared expert output to the final result.
        r6   r!   )kr   r   r   )Zbinsminmaxr   )r?   viewrz   r   r,   ZtopkrU   Zmoe_topkr$   r   softmaxr7   Zhistcflattenr8   r9   r   ZargsortZindex_selectr   rp   r   Zindex_copy_rw   sumr   )r/   r<   Zoriginal_shapelogitsZ
top_logitsZtop_indicesZscoresZoriginal_dtyper   indicesZflatten_indicesZsorted_indicesr   Zexpert_outputZunpermuted_tokensr   Zshared_expert_outputr4   r4   r5   r=   }  s:   



zAriaTextMoELayer.forward)
rB   rC   rD   rS   r#   r*   r,   r}   r=   rE   r4   r4   r2   r5   r   j  s    
r   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr6   r   r   )r?   r,   r   )r   x1Zx2r4   r4   r5   rotate_half  s   r   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )rw   r   )qr   cossinposition_idsZunsqueeze_dimZq_embedZk_embedr4   r4   r5   apply_rotary_pos_emb  s
   

r   r<   n_repr   c                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)r?   ry   reshape)r<   r   batchnum_key_value_headsslenhead_dimr4   r4   r5   	repeat_kv  s
   0r           modulerg   rh   ri   attention_maskscalingr`   c                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr   r   r6   )r   r7   )ptrainingr!   )r   num_key_value_groupsr,   r   	transposer?   r$   r   r   r9   r8   r7   r`   r   
contiguous)r   rg   rh   ri   r   r   r`   kwargs
key_statesvalue_statesattn_weightscausal_maskrj   r4   r4   r5   eager_attention_forward  s   
&r   c                       s   e Zd ZdZdedef fddZ		ddejde	ejejf d	e
ej d
e
e de
ej dee de	eje
ej e
e	ej  f fddZ  ZS )AriaTextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrU   	layer_idxc                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _d S )Nr   g      TrH   )r)   r*   rU   r   getattrr0   rX   r   r   r   r   attention_dropoutZ	is_causalr$   rJ   Zattention_biasrZ   r[   r\   o_projr/   rU   r   r2   r4   r5   r*     s(   
zAriaTextAttention.__init__Nr<   position_embeddingsr   past_key_valuecache_positionr   r   c                 K   sH  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkrw| jjdkrq|ddrqtd	 nt| jj }|| |	|
||f| jsd
n| j| jd|\}}|jg |dR   }| |}||fS )Nr6   r!   r   )r   r   r   eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r   )r`   r   )r?   r   rZ   r   r   r[   r\   r   updater   r   rU   _attn_implementationgetloggerwarning_oncer   r   r   r   r   r   r   )r/   r<   r   r   r   r   r   Zinput_shapeZhidden_shapeZquery_statesr   r   r   r   Zcache_kwargsZattention_interfacerj   r   r4   r4   r5   r=     s@   	

zAriaTextAttention.forward)NN)rB   rC   rD   rS   r#   intr*   r,   r}   r   r   r
   
LongTensorr   r   r=   rE   r4   r4   r2   r5   r     s(    r   c                       s   e Zd ZdZdedef fddZ							ddejd	e	ej d
e	ej
 de	e de	e de	e de	ej
 de	eejejf  dee deeje	eejejf  f fddZ  ZS )AriaTextDecoderLayerag  
    Aria Text Decoder Layer.

    This class defines a single decoder layer in the language model, incorporating self-attention and Mixture of Experts (MoE) feed-forward network.

    Args:
        config (`AriaTextConfig`):
            Configuration object for the text component of the model.
        layer_idx (`int`):
            Index of the layer.
    rU   r   c                    sR   t    |j| _t||d| _t|| _t|j|jd| _	t|j|jd| _
d S )N)rU   r   r1   )r)   r*   r0   r   	self_attnr   mlpr'   rms_norm_epsinput_layernormpost_attention_layernormr   r2   r4   r5   r*   [  s   

zAriaTextDecoderLayer.__init__NFr<   r   r   r   r   	use_cacher   r   r   r   c	                 K   st   |}
|  |}| jd||||||||d|	\}}|
| }|}
| |}| |}|
| }|f}|r8||f7 }|S )N)r<   r   r   r   r   r   r   r   r4   )r   r   r   r   )r/   r<   r   r   r   r   r   r   r   r   ZresidualZself_attn_weightsoutputsr4   r4   r5   r=   d  s.   
	



zAriaTextDecoderLayer.forward)NNNFFNN)rB   rC   rD   rS   r#   r   r*   r,   r}   r   r   r
   boolr   r   r   FloatTensorr=   rE   r4   r4   r2   r5   r   N  s>    	
r   c                   @   s<   e Zd ZeZdZddgZdZdZdZ	dZ
dZdZdd Zd	S )
AriaTextPreTrainedModelmodelr   r   Tpast_key_valuesFc                 C   s   | j j}t|tjr"|jjjd|d |jd ur |jj	  d S d S t|tj
rC|jjjd|d |jd urA|jj|j 	  d S d S t|trQ|jjd d S t|tra|jjjd|d d S d S )Nr   r;   std      ?)rU   initializer_range
isinstancer$   rJ   r-   datanormal_rI   zero_	Embeddingpadding_idxr'   fill_r   r/   r   r   r4   r4   r5   _init_weights  s    



z%AriaTextPreTrainedModel._init_weightsN)rB   rC   rD   r#   config_classbase_model_prefix_no_split_modulessupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_cache_class_supports_attention_backendr   r4   r4   r4   r5   r     s    r   c                   @   sH   e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdZdZdd ZdS )	AriaPreTrainedModel TZAriaDecoderLayerr   Fc                 C   s   | j j}t|tjr"|jjjd|d |jd ur |jj	  d S d S t|tj
r.|  d S t|tjrC|jjd |jj	  d S t|trStjj|j|d d S d S )Nr   r   r   )r   )rU   r   r   r$   rJ   r-   r   r   rI   r   r]   Z_reset_parametersra   r   rm   initZtrunc_normal_rg   r   r4   r4   r5   r     s   

z!AriaPreTrainedModel._init_weightsN)rB   rC   rD   r"   r   r   r   r   r   r  r  Z_supports_flex_attnr  Z_supports_quantized_cacheZ_supports_static_cacher  r   r4   r4   r4   r5   r    s    r  c                       s8   e Zd Zddef fddZe edd Z  Z	S )AriaTextRotaryEmbeddingNrU   c                    s   t    t|dr|jd ur|jd|jd| _nd| _|j| _|j| _|| _	t
| j | _| | j	|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultinv_freqF)
persistent)r)   r*   hasattrr	  r   r
  Zmax_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenrU   r   Zrope_init_fnattention_scalingZregister_bufferr  Zoriginal_inv_freq)r/   rU   r   r  r2   r4   r5   r*     s   
z AriaTextRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r6   r!   Zmpsr   F)device_typeenabledr   r   )r7   )r  rl   ry   r?   r8   r   r   r  strr,   Zautocastr   r   r   r  r   r7   )
r/   r   r   Zinv_freq_expandedZposition_ids_expandedr  ZfreqsZembr   r   r4   r4   r5   r=     s   0&zAriaTextRotaryEmbedding.forwardrQ   )
rB   rC   rD   r#   r*   r,   Zno_gradr   r=   rE   r4   r4   r2   r5   r    s
    r  c                       s  e Zd Zdef fddZdd Zdd Zee									d!d	e	e
j d
e	e
j de	e
j de	e de	e
j de	e de	e de	e de	e
j dee defddZ	d"d
ee
jdf de
jde
jdedef
ddZed
e
jdedede
jde
jdefdd Z  ZS )#AriaTextModelrU   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _|   d S )Nc                    s   g | ]}t  |qS r4   )r   ).0r   rU   r4   r5   
<listcomp>  s    z*AriaTextModel.__init__.<locals>.<listcomp>r   r  F)r)   r*   Zpad_token_idr   
vocab_sizer$   r   r0   embed_tokensZ
ModuleListr   num_hidden_layerslayersr'   r   normr  
rotary_embgradient_checkpointing	post_initrs   r2   r  r5   r*     s   zAriaTextModel.__init__c                 C      | j S rQ   r  r@   r4   r4   r5   get_input_embeddings     z"AriaTextModel.get_input_embeddingsc                 C   
   || _ d S rQ   r!  r/   ri   r4   r4   r5   set_input_embeddings     
z"AriaTextModel.set_input_embeddingsN	input_idsr   r   r   inputs_embedsr   r   output_hidden_statesr   flash_attn_kwargsr   c
                 K   s  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|d u |d uA r*td| jr9| jr9|r9td d}t	|t
d tfsFtd|d u rO| |}|rX|d u rXt }|	d u rt|d urd| nd}tj|||jd  |jd}	|d u r}|	d}| |||	||}|}| ||}|rdnd }|rdnd }| jd | j j D ]&}|r||f7 }||f||||||	|d	|
}|d }|r||d f7 }q| |}|r||f7 }t||r|nd ||d
S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBThe `past_key_values` should be either a `Cache` object or `None`.r   r!   r   r4   )r   r   r   r   r   r   r   )last_hidden_stater   r<   
attentions)rU   r   r*  r   
ValueErrorr  r   r   r   r   r  r
   r  r   get_seq_lengthr,   aranger?   r   rw   _update_causal_maskr  r  r  r  r   )r/   r(  r   r   r   r)  r   r   r*  r   r+  past_seen_tokensr   r<   r   Zall_hidden_statesZall_self_attnsZdecoder_layerZlayer_outputsr4   r4   r5   r=     sx   



	


zAriaTextModel.forwardFr%   input_tensorc                 C   s:  | j jdkr|d ur|dk r|S d S | j jdkr&t|tjr$t|}|S |d ur.| nd}|d ur7|jnd}| j jdkrO|sO|sOt	j
|||| jdrOd S |j}|jd }	|r^| }
nt|tjri|jd	 n||	 d }
| j||	|
|||jd d
}| j jdkr|d ur|jjdv r|st|j}t	||}|S )NZflash_attention_2r   Zflex_attentionr   Fr   )r)  Zpast_key_values_lengthZis_trainingr!   r6   )sequence_lengthtarget_lengthr7   r   r{   )cudaZxpuZnpu)rU   r   anyr   r,   r}   r&   r0  Zis_compileabler   Z_ignore_causal_mask_sdpar   r7   r?   Zget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr   r  finfor   Z_unmask_unattended)r/   r   r4  r   r   r   r3  Zusing_compilable_cacher7   r5  r6  r   	min_dtyper4   r4   r5   r2  d  sT   




z!AriaTextModel._update_causal_maskr5  r6  r7   r{   c                 K   D  | dur|   dkr| }|S t|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9 }|ddddddf 	|ddd}| dur|
 }| jd }	|ddddddd|	f | ddddddf |j }
|
dk}
|ddddddd|	f |
||ddddddd|	f< |S 	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )Z
fill_valuer7   r   r!   )Zdiagonalr,  r6   r   r   r,   r:  r   fullr   Ztriur1  r   ry   cloner?   r8   Zmasked_fillr   r5  r6  r7   r   r{   r   r   r;  Zmask_lengthZpadding_maskr4   r4   r5   r9    ,    $
6  zCAriaTextModel._prepare_4d_causal_attention_mask_with_cache_position)	NNNNNNNNN)F)rB   rC   rD   r#   r*   r"  r&  r   r   r   r,   r   r}   r
   r   r   r   r   r   r=   r   r2  staticmethodr   r7   r9  rE   r4   r4   r2   r5   r    s    	
d
Dr  c                   @   s   e Zd ZdS )KwargsForCausalLMN)rB   rC   rD   r4   r4   r4   r5   rE    s    rE  c                       s  e Zd ZdgZddiZddgdgfiZdef fddZd	d
 Zdd Z	dd Z
dd Zdd Zdd Ze											d&deej deej deej dee deej deej dee dee dee d eej d!eeejf d"ee d#efd$d%Z  ZS )'AriaTextForCausalLMlm_head.weightlm_headZcolwise_repr<   r   rU   c                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S r   )
r)   r*   r  r   r  r$   rJ   r0   rH  r  rs   r2   r4   r5   r*     s
   
zAriaTextForCausalLM.__init__c                 C      | j jS rQ   r   r  r@   r4   r4   r5   r"    s   z(AriaTextForCausalLM.get_input_embeddingsc                 C   s   || j _d S rQ   rJ  r%  r4   r4   r5   r&    s   z(AriaTextForCausalLM.set_input_embeddingsc                 C   r   rQ   rH  r@   r4   r4   r5   get_output_embeddings  r#  z)AriaTextForCausalLM.get_output_embeddingsc                 C   r$  rQ   rK  r/   Znew_embeddingsr4   r4   r5   set_output_embeddings  r'  z)AriaTextForCausalLM.set_output_embeddingsc                 C   r$  rQ   r   )r/   decoderr4   r4   r5   set_decoder  r'  zAriaTextForCausalLM.set_decoderc                 C   r   rQ   rO  r@   r4   r4   r5   get_decoder  r#  zAriaTextForCausalLM.get_decoderNr   r(  r   r   r   r)  labelsr   r   r*  r   logits_to_keepr   r   c                 K   s   |dur|n| j j}|	dur|	n| j j}	| jd||||||||	|
d	|}|j}t|tr4t| dn|}| |dd|ddf }d}|durX| j	d||| j j
d|}t|||j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, AriaTextForCausalLM

        >>> model = AriaTextForCausalLM.from_pretrained("meta-aria_text/AriaText-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-aria_text/AriaText-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)	r(  r   r   r   r)  r   r   r*  r   r   rS  r  lossr   r   r<   r.  r4   )rU   r   r*  r   r-  r   r   slicerH  loss_functionr  r   r   r<   r.  )r/   r(  r   r   r   r)  rS  r   r   r*  r   rT  r   r   r<   slice_indicesr   rW  r4   r4   r5   r=     s:   &
zAriaTextForCausalLM.forward)NNNNNNNNNNr   )rB   rC   rD   _tied_weights_keysZ_tp_planZ_pp_planr#   r*   r"  r&  rL  rN  rQ  rR  r   r   r,   r   r}   r
   r   r   r   r   r   rE  r   r=   rE   r4   r4   r2   r5   rF    sd    		
rF  c                   @   s   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dZeej ed< dS )	AriaCausalLMOutputWithPasta  
    Base class for Aria causal language model (or autoregressive) outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`torch.FloatTensor`, *optional*):
            A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
            image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NrW  r   r   r<   r.  image_hidden_states)rB   rC   rD   rS   rW  r   r,   r   __annotations__r   r   r   r<   r   r.  r]  r4   r4   r4   r5   r\  O  s   
 r\  c                   @   s$   e Zd ZU dZdZeej ed< dS )AriaModelOutputWithPasta  
    Base class for Aria outputs, with hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`torch.FloatTensor`, *optional*):
            A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
            image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nr]  )	rB   rC   rD   rS   r]  r   r,   r   r^  r4   r4   r4   r5   r_  w  s   
 r_  zt
    The Aria model which consists of a vision backbone and a language model, without a language modeling head.
    )Zcustom_introc                !       s  e Zd ZddiZdef fddZdd Zdd	 Z	
	d!dej	de
ej	 defddZee	
	
	
	
	
	
	
	
	
	
	
	
d"dejdej	dejde
ej de
ej de
eej	  de
ej	 de
e de
e de
e de
e de
ej dee deeef fddZdd  Z  ZS )#	AriaModelzlanguage_model.modellanguage_modelrU   c                    s>   t  | t|j| _t|| _t|j| _	| 
  d S rQ   )r)   r*   r    Zfrom_configrW   vision_towerrm   multi_modal_projectorro   ra  r  rs   r2   r4   r5   r*     s
   
zAriaModel.__init__c                 C   
   | j  S rQ   )ra  r"  r@   r4   r4   r5   r"    r'  zAriaModel.get_input_embeddingsc                 C      | j | d S rQ   )ra  r&  r%  r4   r4   r5   r&       zAriaModel.set_input_embeddingsNr6   pixel_values
pixel_maskvision_feature_layerc           
      C   sj   |dur|n| j j}| |}| j||dd}d}|dur'|d}t|}|j| }| j||d}	|	S )aM  
        Obtains image last hidden states from the vision tower and apply multimodal projection.

        Args:
            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`):
               The tensors corresponding to the input images.
            pixel_mask (`torch.FloatTensor]`, *optional*):
                The tensors corresponding to the input image mask.
            vision_feature_layer (`Union[int, List[int]]`, *optional*):
                The index of the layer to select the vision feature. If multiple indices are provided,
                the vision feature of the corresponding indices will be concatenated to form the
                vision features.
        Returns:
            image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
        NT)patch_attention_maskr*  r!   rd   )	rU   ri  _create_patch_attention_maskrb  r   r,   Zlogical_notr<   rc  )
r/   rg  rh  ri  rj  Zimage_outputsZimage_attn_maskZflattened_maskZselected_image_featureimage_featuresr4   r4   r5   get_image_features  s   



zAriaModel.get_image_featuresr(  r   r   r   r)  r   r   r*  return_dictr   r   r   c                 K   s  |	d ur|	n| j j}	|
d ur|
n| j j}
|d ur|n| j j}|d u r(|  |}|d ur|jd dkr|d u rV||  tj| j jtj	|j
dk}|jddjddd }n|| j jk}|d||j
}|jddjdd}| j||| j jd}|jd |jd }}|| }||krtd| d| ||j
|j}|||}| jd||||||	|
d	|d
	|}t|j|r|jnd |j|j|d ur|dS d dS )Nr!   r   r   r   r6   )rg  rh  ri  z6Image features and image tokens do not match: tokens: z, features T)	r   r   r   r)  r   r   r*  rn  r   )r-  r   r<   r.  r]  r4   )rU   r   r*  use_return_dictr"  r?   r,   ZtensorZimage_token_idr   r   r   rw   Z	expand_asr8   rm  ri  r/  r7   Zmasked_scatterra  r_  r-  r   r<   r.  )r/   r(  rg  rh  r   r   r   r)  r   r   r*  rn  r   r   Zspecial_image_maskZn_image_tokensZimage_embedsrl  Zn_imagesZn_features_per_imageZn_image_featuresr   r4   r4   r5   r=     sd   

zAriaModel.forwardc                 C   sX   |d u rd S |j d| jjj| jjjd}|j d| jjj| jjjd}|jdddk S )Nr!   )	dimensionrz   stepr   )r6   r   r   r   )Zunfoldrb  rU   Z
patch_sizer   r   )r/   rh  Zpatches_subgridr4   r4   r5   rk    s   z&AriaModel._create_patch_attention_mask)Nr6   )NNNNNNNNNNNN)rB   rC   rD   _checkpoint_conversion_mappingr"   r*   r"  r&  r,   r   r   r   rm  r   r   r   r}   r   r   r   r   r   r   r_  r=   rk  rE   r4   r4   r2   r5   r`    st    
%	

Hr`  z
    Aria model for conditional generation tasks.

    This model combines a vision tower, a multi-modal projector, and a language model
    to perform tasks that involve both image and text inputs.
    c                %       s  e Zd ZdddddZdgZdef fdd	Zd
d Zdd Zde	j
fddZdd Zedd Zedd Zedd Zee														d4dejdejdejdeej deej d eeej  d!eej d"eej d#ee d$ee d%ee d&ee d'eeejf d(eej d)ee deeef f d*d+Z 							d5 fd,d-	Z!e"dejd.ed/ed0ej#d(ejd1efd2d3Z$  Z%S )6AriaForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorrH  )z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headrG  rU   c                    s<   t  | t|| _tj|jj|jjdd| _	| 
  d S r   )r)   r*   r`  r   r$   rJ   ro   r0   r  rH  r  rs   r2   r4   r5   r*   >  s   
z%AriaForConditionalGeneration.__init__c                 C   rd  rQ   )r   r"  r@   r4   r4   r5   r"  D  r'  z1AriaForConditionalGeneration.get_input_embeddingsc                 C   re  rQ   )r   r&  r%  r4   r4   r5   r&  G  rf  z1AriaForConditionalGeneration.set_input_embeddingsr   c                 C   r   rQ   rK  r@   r4   r4   r5   rL  J  r#  z2AriaForConditionalGeneration.get_output_embeddingsc                 C   r$  rQ   rK  rM  r4   r4   r5   rN  M  r'  z2AriaForConditionalGeneration.set_output_embeddingsc                 C   rI  rQ   )r   ra  r@   r4   r4   r5   ra  Q     z+AriaForConditionalGeneration.language_modelc                 C   rI  rQ   )r   rb  r@   r4   r4   r5   rb  U  rt  z)AriaForConditionalGeneration.vision_towerc                 C   rI  rQ   )r   rc  r@   r4   r4   r5   rc  Y  rt  z2AriaForConditionalGeneration.multi_modal_projectorNr   r(  rg  rh  r   r   r   r)  rS  r   r   r*  rn  rT  r   r   c                 K   s   |
dur|
n| j j}
|dur|n| j j}|dur|n| j j}| jd||||||||	|
|||d|}|d }t|trBt| dn|}| |dd|ddf }d}|durg| j	d||| j j
jd|}t|||j|j|jdS )aa  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `AriaForConditionalGeneration`).
            Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only
            computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> import requests
        >>> import torch
        >>> from PIL import Image
        >>> from io import BytesIO

        >>> from transformers import AutoProcessor, AutoModel
        >>> from transformers.image_utils import load_image

        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")

        >>> processor = AutoProcessor.from_pretrained("Rhymes-AI/Aria")
        >>> model = AutoModel.from_pretrained("Rhymes-AI/Aria", torch_dtype=torch.bfloat16, device_map="auto")

        >>> # Create inputs
        >>> messages = [
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "image"},
        ...             {"type": "text", "text": "In this image, we can see the city of New York, and more specifically the Statue of Liberty."},
        ...             {"type": "image"},
        ...             {"type": "text", "text": "What can we see in this image?"},
        ...         ]
        ...     },
        ...     {
        ...         "role": "user",
        ...         "content": [
        ...             {"type": "image"},
        ...             {"type": "text", "text": "In which city is that bridge located?"},
        ...         ]
        ...     }
        ... ]

        >>> prompts = [processor.apply_chat_template([message], add_generation_prompt=True) for message in messages]
        >>> images = [[image1, image2], [image3]]
        >>> inputs = processor(text=prompts, images=images, padding=True, return_tensors="pt").to(model.device)

        >>> # Generate
        >>> generated_ids = model.generate(**inputs, max_new_tokens=256)
        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)

        >>> print(generated_texts[0])
        Assistant: There are buildings, trees, lights, and water visible in this image.

        >>> print(generated_texts[1])
        Assistant: The bridge is in San Francisco.
        ```N)r(  rg  rh  r   r   r   r)  r   r   r*  rn  r   r   rU  rV  r4   )rU   r   r*  ro  r   r   r   rX  rH  rY  ro   r  r\  r   r<   r.  )r/   r(  rg  rh  r   r   r   r)  rS  r   r   r*  rn  rT  r   r   r   r<   rZ  r   rW  r4   r4   r5   r=   ]  sJ   Pz$AriaForConditionalGeneration.forwardc	                    s@   t  j|f|||||d|	}
|d dkr||
d< ||
d< |
S )N)r   r)  r   r   rT  r   rg  rh  )r)   prepare_inputs_for_generation)r/   r(  r   r)  rg  rh  r   r   rT  r   Zmodel_inputsr2   r4   r5   ru    s   
z:AriaForConditionalGeneration.prepare_inputs_for_generationr5  r6  r7   r{   c                 K   r<  r=  r?  rB  r4   r4   r5   r9    rC  zRAriaForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_position)NNNNNNNNNNNNr   N)NNNNNNN)&rB   rC   rD   rr  r[  r"   r*   r"  r&  r$   ModulerL  rN  propertyra  rb  rc  r   r   r,   r   r   r   r}   r   r   r   r   r   rE  r   r\  r=   ru  rD  r7   r9  rE   r4   r4   r2   r5   rs  -  s    



	

zrs  )rs  r  r   r  r`  rF  )Nr!   )r   )Wdataclassesr   typingr   r   r   r   r   Zactivationsr	   Zcache_utilsr
   r   Z
generationr   Zintegrationsr   Zmodeling_attn_mask_utilsr   Zmodeling_flash_attention_utilsr   Zmodeling_layersr   Zmodeling_outputsr   r   r   Zmodeling_rope_utilsr   r   Zmodeling_utilsr   r   Zprocessing_utilsr   utilsr   r   r   r   r   Zutils.import_utilsr   autor    Zconfiguration_ariar"   r#   r,   r$   Z!torch.nn.attention.flex_attentionr%   Zintegrations.flex_attentionr&   Z
get_loggerrB   r   rv  r'   rG   rT   rm   r~   r   r   r   r   r   r   r}   r   r   rl   r   r   r   r   r  r  r  rE  rF  r\  r_  r`  rs  __all__r4   r4   r4   r5   <module>   s   
7A,"O

M@" tk'   x