o
    Zh                     @   sh  d Z ddlZddlmZmZmZ ddlZddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ e$ rddl,m-Z- ddl.m/Z/ e rddlm0Z0 e&1e2Z3de4de4dej5fddZ6ej7j8dd Z9dej5dej5fddZ:dej5d ej5d!ej5dej5fd"d#Z;G d$d% d%e	j<Z=G d&d' d'e=Z>e=e>d(Z?G d)d* d*e	j<Z@G d+d, d,e	j<ZAe#G d-d. d.e ZBd/ZCd0ZDe#G d1d2 d2eBZEe#d3d4G d5d6 d6eBeZFe#d7d4G d8d9 d9eBZGe#G d:d; d;eBZHg d<ZIdS )=zPyTorch GPT-J model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)!flash_attn_supports_top_left_maskis_flash_attn_available)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPast)PreTrainedModel)add_start_docstringsauto_docstringis_torch_flex_attn_availableis_torch_fx_proxylogging)assert_device_mapget_device_map   )
GPTJConfig)	BlockMask)make_flex_block_causal_mask)_flash_attention_forwardnum_posdimreturnc                 C   s`   ddt jd|dt jd|   }t dt j| t jd | }t jt |t |fddS )	N      ?i'  r      dtypezi , j -> i jr   r#   )torcharangeZint64Zeinsumfloatcatsincos)r"   r#   Zinv_freqZsinusoid_inp r0   U/var/www/auris/lib/python3.10/site-packages/transformers/models/gptj/modeling_gptj.pycreate_sinusoidal_positions>   s    "r2   c                 C   s   |  |j|jd ddS Nr   r   )todevicerepeatshape)embed_positionsposition_idsr0   r0   r1   get_embed_positionsD   s   r:   xc                 C   sb   | d d d d d d d d df }| d d d d d d dd df }t j| |fdd} | dS )Nr&   r   r)   )r*   stackflatten)r;   x1Zx2r0   r0   r1   rotate_every_twoI   s   ""
rA   tensorr.   r/   c                 C   s`   t |d d d d d d d f dd}t |d d d d d d d f dd}| | t| |  S )Nr&   r	   )r*   Zrepeat_interleaverA   )rB   r.   r/   r0   r0   r1   apply_rotary_pos_embP   s   &&rC   c                       s   e Zd Zd fdd	Zdd Zdd Z		ddd	Zd
d Z							ddej	de
e de
ej	 de
ej de
ej	 de
e de
e de
ej deeejeej f e
eejeej eejdf f  f fddZ  ZS )GPTJAttentionNc                    sF  t    || _|j}t|j| _t|j| _	d| _
|| _|d u r.td| jj d |j| _|j| _| j| j | _| j| j | jkrStd| j d| j dttj| jtjdt | _tj| j| jdd	| _tj| j| jdd	| _tj| j| jdd	| _tj| j| jdd	| _ |j!| _!| j!p| j}t"||| _#d S )
NTzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r'   Fbias)$super__init__configZmax_position_embeddingsr   Dropout
attn_pdropattn_dropoutresid_pdropresid_dropout	is_causal	layer_idxloggerwarning_once	__class____name__hidden_size	embed_dimnum_attention_headshead_dim
ValueErrorr*   sqrtrB   float32r4   Zget_default_dtype
scale_attnLineark_projv_projq_projout_proj
rotary_dimr2   r8   )selfrI   rP   Zmax_positionsZpos_embd_dimrS   r0   r1   rH   W   s8   

$zGPTJAttention.__init__c                 C   s|   |  dd ||f }||}|r|S t|jdkr%|dddddS t|jdkr4|ddddS td	t|j )
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        Nr<      r   r   r	   r&      3Input tensor rank should be one of [4, 5], but is: )sizeviewlenr7   permuterY   )rc   rB   rW   attn_head_sizeZrotary	new_shaper0   r0   r1   _split_headsz   s   
zGPTJAttention._split_headsc                 C   s   t |jdkr|ddddd }nt |jdkr%|dddd }n
tdt |j | dd	 || f }||S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        re   r   r   r	   r&   rf   rg   Nr=   )rj   r7   rk   
contiguousrY   rh   ri   )rc   rB   rW   rl   rm   r0   r0   r1   _merge_heads   s   
zGPTJAttention._merge_headsc           	      C   s   | tj}| tj}t||dd}|| j }|d ur6|d d d d d d d |jd f }|| }tjj	|dd}| |j
}| |}|d urQ|| }t||}||fS )Nr<   r=   r)   )r4   r*   r[   matmulZ	transposer\   r7   r   Z
functionalZsoftmaxr(   rL   )	rc   querykeyvalueattention_mask	head_maskattn_weightscausal_maskattn_outputr0   r0   r1   _attn   s   	
&
zGPTJAttention._attnc                 C   s8   | j }|j|jkr||j}|| _ ||jd ddS r3   )r8   r5   r4   r6   r7   )rc   r9   r8   r0   r0   r1   _get_embed_positions   s
   z"GPTJAttention._get_embed_positionsFhidden_states
layer_pastru   r9   rv   	use_cacheoutput_attentionscache_positionr$   .c	                 C   sZ  |  |}	| |}
| |}| |	| j| jd}	| |
| j| jd}
| || j| jd}t|s6tj	 r=t
| j|}n| |}|ddd|jd }t|d|}tj||jd d dd\}}| jd ur|
d d d d d d d | jf }|
d d d d d d | jd f }|	d d d d d d d | jf }|	d d d d d d | jd f }t|||}t|||}tj||gdd}
tj||gdd}	nt|
||}
t|	||}	|
dddd}
|	dddd}	|d ur||| j|d	}||
|| j|\}
}| |	|
|||\}}| || j| j}| |}| |}||f}|r+||f7 }|S )
NTFr<   r   r&   r)   r   r	   r.   r/   Zpartial_rotation_sizer   )r`   r^   r_   rn   rW   rX   r   r*   jit
is_tracingr:   r8   r{   	unsqueezer6   r7   gathersplitrb   rC   r-   rk   updaterP   rz   rp   ra   rN   )rc   r|   r}   ru   r9   rv   r~   r   r   rr   rs   rt   r8   repeated_position_idssincosr.   r/   k_rotk_passq_rotq_passcache_kwargsry   rw   outputsr0   r0   r1   forward   sP   




""""



zGPTJAttention.forwardN)NNNNNNFFN)rT   
__module____qualname__rH   rn   rp   rz   r{   r*   FloatTensorr   r   
LongTensorboolr   r   Tensorr   __classcell__r0   r0   rd   r1   rD   V   sL    #

	
"rD   c                       s   e Zd ZdZ fddZ							ddejdee deej d	eej	 d
eej dee
 dee
 deej	 deeejeej f eeejeej eejdf f  f fddZ  ZS )GPTJFlashAttention2aD  
    GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                    s   t  j|i | t | _d S r   )rG   rH   r   _flash_attn_uses_top_left_mask)rc   argskwargsrd   r0   r1   rH     s   zGPTJFlashAttention2.__init__NFr|   r}   ru   r9   rv   r~   r   r   r$   .c	              
   C   sJ  |  |}	| |}
| |}| |	| j| jd}	| |
| j| jd}
| || j| jd}t|s6tj	 r=t
| j|}n| |}|ddd|jd }t|d|}tj||jd d dd\}}| jd ur|
d d d d d d d | jf }|
d d d d d d | jd f }|	d d d d d d d | jf }|	d d d d d d | jd f }t|||}t|||}tj||gdd}
tj||gdd}	nt|
||}
t|	||}	|
dddd}
|	dddd}	|d ur||| j|d	}||
|| j|\}
}|
dddd }
|	dddd }	|dddd }|	j}|tjkr[t r2t }nt| jd
r>| jj}n| j j j}t!"d| d |	#|}	|
#|}
|#|}| j$rc| jj%nd}|	jd }t&|	|
||||| j'| j(d}|)|jd |jd |jd |jd  }| *|}| +|}||f}|r||f7 }|S )NTFr<   r   r&   r)   r   r	   r   _pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .        )dropoutrO   Zuse_top_left_mask),r`   r^   r_   rn   rW   rX   r   r*   r   r   r:   r8   r{   r   r6   r7   r   r   rb   rC   r-   rk   r   rP   ro   r(   r[   Zis_autocast_enabledZget_autocast_gpu_dtypehasattrrI   r   weightrQ   rR   r4   trainingrK   r!   rO   r   reshapera   rN   )rc   r|   r}   ru   r9   rv   r~   r   r   rr   rs   rt   r8   r   r   r.   r/   r   r   r   r   r   Zinput_dtypeZtarget_dtypeZattention_dropoutZquery_lengthrw   ry   r   r0   r0   r1   r     s   




""""








"


zGPTJFlashAttention2.forwardr   )rT   r   r   __doc__rH   r*   r   r   r   r   r   r   r   r   r   r   r0   r0   rd   r1   r     sB    	
"r   )eagerflash_attention_2c                       s6   e Zd Z fddZdeej dejfddZ  ZS )GPTJMLPc                    sJ   t    |j}t||| _t||| _t|j | _	t
|j| _d S r   )rG   rH   n_embdr   r]   fc_infc_outr
   Zactivation_functionactrJ   rM   r   )rc   Zintermediate_sizerI   rV   rd   r0   r1   rH     s   
zGPTJMLP.__init__r|   r$   c                 C   s,   |  |}| |}| |}| |}|S r   )r   r   r   r   )rc   r|   r0   r0   r1   r     s
   



zGPTJMLP.forward)	rT   r   r   rH   r   r*   r   r   r   r0   r0   rd   r1   r     s    "
r   c                       s   e Zd Zd fdd	Z							ddeej dee deej deej d	eej d
ee	 dee	 deej de
eej eeejeejdf f  f fddZ  ZS )	GPTJBlockNc                    sZ   t    |jd ur|jnd|j }tj|j|jd| _t|j	 ||| _
t||| _d S )Nrf   Zeps)rG   rH   Zn_innerr   r   	LayerNormlayer_norm_epsilonln_1GPTJ_ATTENTION_CLASSES_attn_implementationattnr   mlp)rc   rI   rP   Z	inner_dimrd   r0   r1   rH     s
   
zGPTJBlock.__init__Fr|   r}   ru   r9   rv   r~   r   r   r$   .c	              
   C   sz   |}	|  |}| j||||||||d}
|
d }|
dd  }| |}|| |	 }|r2|f| }|S |f|dd   }|S )Nr|   r}   ru   r9   rv   r~   r   r   r   r   )r   r   r   )rc   r|   r}   ru   r9   rv   r~   r   r   ZresidualZattn_outputsry   r   Zfeed_forward_hidden_statesr0   r0   r1   r     s*   



zGPTJBlock.forwardr   r   )rT   r   r   rH   r   r*   r   r   r   r   r   r   r   r   r   r0   r0   rd   r1   r     s8    
	(
r   c                       sR   e Zd ZeZdZdZdZdgZdZ	dZ
dZdZdZdZ fddZdd	 Z  ZS )
GPTJPreTrainedModeltransformerTr   past_key_valuesFc                    s   t  j|i | d S r   )rG   rH   )rc   Zinputsr   rd   r0   r1   rH     s   zGPTJPreTrainedModel.__init__c                 C   s   t |tjfr!|jjjd| jjd |jdur|jj	  dS dS t |tj
rD|jjjd| jjd |jdurB|jj|j 	  dS dS t |tjrY|jj	  |jjd dS dS )zInitialize the weights.r   )meanZstdNr%   )
isinstancer   r]   r   dataZnormal_rI   Zinitializer_rangerF   Zzero_	EmbeddingZpadding_idxr   Zfill_)rc   moduler0   r0   r1   _init_weights  s   

z!GPTJPreTrainedModel._init_weights)rT   r   r   r   Zconfig_classZbase_model_prefixZis_parallelizableZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementZ_supports_flash_attn_2Z_supports_cache_classZ_supports_quantized_cacheZ_supports_static_cacheZ!_supports_param_buffer_assignmentrH   r   r   r0   r0   rd   r1   r     s    r   a  
    This is an experimental feature and is a subject to change at a moment's notice. Uses a device map to distribute
    attention modules of the model across several devices. If no device map is given, it will evenly distribute blocks
    across all devices.

    Args:
        device_map (`Dict[int, list]`, *optional*):
            A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are always
            automatically mapped to the first device (for esoteric reasons). That means that the first device should
            have fewer attention modules mapped to it than other devices. For reference, the GPT-J models have the
            following number of attention modules:

                - gpt-j-6B: 28

    Example:

    ```python
    # Here is an example of a device map on a machine with 4 GPUs using gpt-j-6B, which has a total of 28 attention modules:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)
    ```
a6  
    Moves the model to CPU from a model parallel state.

    Example:

    ```python
    # On a 4 GPU machine with gpt-j-6B:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)  # Splits the model across several devices
    model.deparallelize()  # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
    ```
c                       s^  e Zd Z fddZeed&ddZeedd Zdd	 Z	d
d Z
e												d'deej deeeeeej  f  deej deej deej deej deej dee dee dee dee deej deeef fddZ	d(deejdf dejdejdedef
ddZedejd ed!ed"ejdejd#efd$d%Z  ZS ))	GPTJModelc                    s   t     j| _ j| _t j| j| _t j	| _
t fddt jD | _tj| j jd| _d| _d | _d| _|    jdk| _d S )Nc                    s   g | ]}t  |d qS ))rP   )r   ).0irI   r0   r1   
<listcomp>9  s    z&GPTJModel.__init__.<locals>.<listcomp>r   Fr   )rG   rH   r   rV   
vocab_sizer   r   wterJ   Z
embd_pdropdropZ
ModuleListrangen_layerhr   r   ln_fmodel_parallel
device_mapgradient_checkpointing	post_initr   Z_use_flash_attention_2rc   rI   rd   r   r1   rH   2  s    zGPTJModel.__init__Nc                 C   s   t dt |d u rtt| jttj	 n|| _
t| j
t| j d| _d| j
 v r/dn
dtt| j
  | _dtt| j
  | _| j| j| _| j
 D ]\}}|D ]}dt| }| j| || j|< qZqT| j| j| _d S )Na6  `GPTJModel.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'h.0': 0, 'h.1': 1, ...}Tcpucuda:)warningswarnFutureWarningr   rj   r   r   r*   cudadevice_countr   r   r   keysstrminfirst_devicemaxlast_devicer   r4   itemsr   )rc   r   kvblockZcuda_devicer0   r0   r1   parallelizeF  s"   $	*zGPTJModel.parallelizec                 C   sx   t dt d| _d | _d| _d| _| jd| _t	t
| jD ]}| j| d| j|< q | jd| _tj  d S )N\Like `parallelize`, `deparallelize` is deprecated and will be removed in v5 of Transformers.Fr   )r   r   r   r   r   r   r   r   r4   r   rj   r   r   r*   r   empty_cache)rc   indexr0   r0   r1   deparallelize`  s   zGPTJModel.deparallelizec                 C      | j S r   r   rc   r0   r0   r1   get_input_embeddingsp     zGPTJModel.get_input_embeddingsc                 C   
   || _ d S r   r   rc   Znew_embeddingsr0   r0   r1   set_input_embeddingss     
zGPTJModel.set_input_embeddings	input_idsr   ru   token_type_idsr9   rv   inputs_embedsr~   r   output_hidden_statesreturn_dictr   r$   c                 C   s  |	dur|	n| j j}	|
dur|
n| j j}
|dur|n| j j}|dur$|n| j j}|du |duA r4td| jrC| jrC|rCt	d d}|du rL| 
|}d}|rit|tsid}|du r_t }n
t|}t	d |jd }|du r|durz| nd}tj||| |jd	}|du r|d}| |||||	}| || j j}|}|dur|d
|}| 
|}|| }| |}d
||d
f}d}|	rdnd}|
rdnd}t| jD ]\}}| jrtj|j |dur|j !|j|_ |j"!|j|_"|dur|!|j}t|tj#r|!|j}|
r||f }| jr2| jr2| $|j%|d|||| ||	|	}n||||||| ||	|d}|d }|du rM|d }|	r\|||rWdnd f }| jr| j&' D ]!\}}||d
 krdt(| | j)kr|!dt(|d  }qeq| *|}||}|
r||f }|r|nd}|r|+ }|st,dd ||||fD S t-||||dS )  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FTzWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)r   r   r5   r<   r0   r   r&   r   c                 s   s    | ]	}|d ur|V  qd S r   r0   )r   r   r0   r0   r1   	<genexpr>  s    z$GPTJModel.forward.<locals>.<genexpr>)Zlast_hidden_stater   r|   
attentions).rI   r   r   r~   use_return_dictrY   r   r   rQ   rR   r   r   r   r   Zfrom_legacy_cacher7   get_seq_lengthr*   r+   r5   r   _update_causal_maskZget_head_maskr   ri   r   rh   	enumerater   r   r   
set_deviceZ	key_cacher4   Zvalue_cacher   Z_gradient_checkpointing_func__call__r   r   r   r   r   Zto_legacy_cachetupler   )rc   r   r   ru   r   r9   rv   r   r~   r   r   r   r   Zreturn_legacy_cacheZ
seq_lengthpast_key_values_lengthrx   r|   Ztoken_type_embedsZoutput_shapeZnext_decoder_cacheZall_self_attentionsZall_hidden_statesr   r   r   r   r   Z
next_cacher0   r0   r1   r   v  s   









"



zGPTJModel.forwardFr   input_tensorc                 C   s:  | j jdkr|d ur|dk r|S d S | j jdkr&t|tjr$t|}|S |d ur.| nd}|d ur7|jnd}| j jdkrO|sO|sOt	j
|||| jdrOd S |j}|jd }	|r^| }
nt|tjri|jd	 n||	 d }
| j||	|
|||jd d
}| j jdkr|d ur|jjdv r|st|j}t	||}|S )Nr   r   Zflex_attentionr   FZsdpa)r   r  Zis_trainingr   r<   )sequence_lengthtarget_lengthr(   r   
batch_size)r   ZxpuZnpu)rI   r   anyr   r*   r   r    r   Zis_compileabler   Z_ignore_causal_mask_sdpar   r(   r7   Zget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr5   typefinfor   Z_unmask_unattended)rc   ru   r  r   r   r   Zpast_seen_tokensZusing_compilable_cacher(   r  r  rx   	min_dtyper0   r0   r1   r     sT   




zGPTJModel._update_causal_maskr  r  r(   r  c                 K   sD  | dur|   dkr| }|S t|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9 }|ddddddf 	|ddd}| dur|
 }| jd }	|ddddddd|	f | ddddddf |j }
|
dk}
|ddddddd|	f |
||ddddddd|	f< |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nrf   )Z
fill_valuer(   r5   r   )Zdiagonalr   r<   r   )r#   r*   r	  r   fullr5   Ztriur+   r   expandcloner7   r4   Zmasked_fill)ru   r  r  r(   r   r  r   rx   r
  Zmask_lengthZpadding_maskr0   r0   r1   r  a  s,    $
6  z?GPTJModel._prepare_4d_causal_attention_mask_with_cache_positionr   NNNNNNNNNNNN)F)rT   r   r   rH   r   PARALLELIZE_DOCSTRINGr   DEPARALLELIZE_DOCSTRINGr   r   r   r   r   r*   r   r   r   r   r   r   r   r   r   r   staticmethodintr(   r  r   r0   r0   rd   r1   r   0  s    
	

 -
Dr   zK
    The GPT-J Model transformer with a language modeling head on top.
    )Zcustom_introc                        sD  e Zd ZdgZ fddZeed ddZeedd Z	d	d
 Z
dd Ze													d!deej deeeeeej  f  deej deej deej deej deej deej dee dee dee dee deej deeef fddZedeeej  dejdeeej  fddZ  ZS )"GPTJForCausalLMzlm_head.weightc                    s@   t  | t|| _t|j|j| _d| _	d | _
|   d S NF)rG   rH   r   r   r   r]   r   r   lm_headr   r   r   r   rd   r0   r1   rH     s   
zGPTJForCausalLM.__init__Nc                 C   st   t dt |d u rtt| jjttj	
 n|| _t| jt| jj | j| j | j| jj| _d| _d S )NaT  `GPTJForCausalLM.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'transformer.h.0': 0, 'transformer.h.1': 1, ...}T)r   r   r   r   rj   r   r   r   r*   r   r   r   r   r   r  r4   r   r   )rc   r   r0   r0   r1   r     s   	
zGPTJForCausalLM.parallelizec                 C   sF   t dt | j  | jd| _| jd| _d| _tj	
  d S )Nr   r   F)r   r   r   r   r   r4   r  r   r*   r   r   r   r0   r0   r1   r     s   
zGPTJForCausalLM.deparallelizec                 C   r   r   r  r   r0   r0   r1   get_output_embeddings  r   z%GPTJForCausalLM.get_output_embeddingsc                 C   r   r   r  r   r0   r0   r1   set_output_embeddings  r   z%GPTJForCausalLM.set_output_embeddingsr   r   ru   r   r9   rv   r   labelsr~   r   r   r   r   r$   c                 K   s   |dur|n| j j}| j||||||||	|
|||d}|d }| jr2tj| jj || j	j
j}| 	|tj}d}|dur[||j}| j||fd| j ji|}||j}|sq|f|dd  }|duro|f| S |S t|||j|j|jdS )aG  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)r   ru   r   r9   rv   r   r~   r   r   r   r   r   r   r   losslogitsr   r|   r   )rI   r   r   r   r*   r   r   r   r4   r  r   r5   r[   Zloss_functionr   r(   r   r   r|   r   )rc   r   r   ru   r   r9   rv   r   r  r~   r   r   r   r   r   transformer_outputsr|   Z	lm_logitsr  outputr0   r0   r1   r     sT   zGPTJForCausalLM.forwardbeam_idxc                    s   t  fdd| D S )a  
        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        c                 3   s&    | ]}t  fd d|D V  qdS )c                 3   s$    | ]}| d  |jV  qdS )r   N)Zindex_selectr4   r5   )r   Z
past_stater  r0   r1   r   /  s   " z;GPTJForCausalLM._reorder_cache.<locals>.<genexpr>.<genexpr>Nr   )r   r}   r   r0   r1   r   .  s
    
z1GPTJForCausalLM._reorder_cache.<locals>.<genexpr>r!  )r   r  r0   r   r1   _reorder_cache%  s   	zGPTJForCausalLM._reorder_cacher   )NNNNNNNNNNNNN)rT   r   r   Z_tied_weights_keysrH   r   r  r   r  r   r  r  r   r   r*   r   r   r   r   r   r   r   r   r   r  r"  r   r0   r0   rd   r1   r    sx    
	

Qr  a  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Z fddZe												ddeej deeeej	   deej
 deej deej d	eej
 d
eej
 deej dee dee dee dee deeef fddZ  ZS )GPTJForSequenceClassificationc                    sL   t  | |j| _t|| _tj|j| jdd| _d| _	d | _
|   d S )NFrE   )rG   rH   
num_labelsr   r   r   r]   r   scorer   r   r   r   rd   r0   r1   rH   C  s   
z&GPTJForSequenceClassification.__init__Nr   r   ru   r   r9   rv   r   r  r~   r   r   r   r$   c                 C   sF  |dur|n| j j}| j||||||||	|
||d}|d }| |}|dur-|jd }n|jd }| j jdu r@|dkr@td| j jdu rId}n1|durn|| j jk|jt	j
}t	j|jd |jt	j
d}|| d}nd}t| jj d |t	j||jd	|f }d}|dur||j}| j jdu r| jdkrd
| j _n| jdkr|jt	jks|jt	jkrd| j _nd| j _| j jd
krt }| jdkr|| | }n+|||}n%| j jdkrt }||d| j|d}n| j jdkrt }|||}|s|f|dd  }|dur|f| S |S t|||j|j|jdS )a!  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)
r   ru   r   r9   rv   r   r~   r   r   r   r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r<   )r5   r(   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r   Z
regressionZsingle_label_classificationZmulti_label_classificationr  )rI   r   r   r%  r7   Zpad_token_idrY   r4   r5   r*   Zint32r+   ZargmaxrQ   rR   rS   rT   Zproblem_typer$  r(   longr  r   squeezer   ri   r   r   r   r|   r   )rc   r   r   ru   r   r9   rv   r   r  r~   r   r   r   r  r|   r  r  Zlast_non_pad_tokenZnon_pad_maskZtoken_indicesZpooled_logitsr  loss_fctr  r0   r0   r1   r   P  s|   



"


z%GPTJForSequenceClassification.forwardr  )rT   r   r   rH   r   r   r*   r   r   r   r   r   r   r   r   r   r0   r0   rd   r1   r#  4  sT    	

r#  c                       s   e Zd Z fddZe											ddeej deej deej deej deej d	eej d
eej deej dee	 dee	 dee	 de
eef fddZ  ZS )GPTJForQuestionAnsweringc                    sH   t  | |j| _t|| _t|j|j| _d| _	d | _
|   d S r  )rG   rH   r$  r   r   r   r]   rU   
qa_outputsr   r   r   r   rd   r0   r1   rH     s   
z!GPTJForQuestionAnswering.__init__Nr   ru   r   r9   rv   r   start_positionsend_positionsr   r   r   r$   c                 C   sX  |dur|n| j j}| j|||||||	|
|d	}|d }| |}|jddd\}}|d }|d }d}|dur|durt| dkrS|d	|j
}t| dkrd|d	|j
}|d}|d|}|d|}t|d}|||}|||}|| d }|s||f|dd  }|dur|f| S |S t||||j|jd	S )
r   N)ru   r   r9   rv   r   r   r   r   r   r   r<   r)   )Zignore_indexr&   )r  start_logits
end_logitsr|   r   )rI   r   r   r*  r   r'  ro   rj   rh   r4   r5   clampr   r   r|   r   )rc   r   ru   r   r9   rv   r   r+  r,  r   r   r   r   Zsequence_outputr  r-  r.  Z
total_lossZignored_indexr(  Z
start_lossZend_lossr  r0   r0   r1   r     sP   




z GPTJForQuestionAnswering.forward)NNNNNNNNNNN)rT   r   r   rH   r   r   r*   r   r   r   r   r   r   r   r   r0   r0   rd   r1   r)    sN    	

r)  )r  r)  r#  r   r   )Jr   r   typingr   r   r   r*   Ztorch.fxZtorch.utils.checkpointr   Ztorch.nnr   r   r   Zactivationsr
   Zcache_utilsr   r   Z
generationr   Zmodeling_attn_mask_utilsr   Zmodeling_flash_attention_utilsr   r   Zmodeling_outputsr   r   r   r   Zmodeling_utilsr   utilsr   r   r   r   r   Zutils.model_parallel_utilsr   r   Zconfiguration_gptjr   Z!torch.nn.attention.flex_attentionr   Zintegrations.flex_attentionr    r!   Z
get_loggerrT   rQ   r  r   r2   Zfxwrapr:   rA   rC   ModulerD   r   r   r   r   r   r  r  r   r  r#  r)  __all__r0   r0   r0   r1   <module>   st   

" 2 -!  k uV