
    fTh                        S SK Jr  S SKrS SKJr  S SKrSSKJr  SSKJ	r	J
r
JrJr  SSKJr  SS	KJr  \R"                  " \5      rS
r " S S\5      rS rSS jr " S S\	5      r " S S\
5      r " S S\5      r " S S\5      r/ SQrg)    )OptionalN   )logging   )LlamaAttentionLlamaForCausalLMLlamaForSequenceClassificationLlamaForTokenClassification)Phi3MLP   )	GlmConfigzTHUDM/glm-4-9bc                       \ rS rSrSrg)GlmMLP&    N__name__
__module____qualname____firstlineno____static_attributes__r       [/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/glm/modular_glm.pyr   r   &       r   r   c                 x    U SSSS24   nU SSSS24   n[         R                  " U* U4SS9R                  S5      $ )	z*Rotates half the hidden dims of the input..r   Nr   r   dim)torchstackflatten)xx1x2s      r   rotate_halfr&   *   sJ    	
319B	
319B;;Ryb)11"55r   c                    UR                  U5      nUR                  U5      nUSSUR                  S   S-  24   R                  SSS9nUSSUR                  S   S-  24   R                  SSS9nUR                  S   nU SSU24   U SUS24   pUSSU24   USUS24   pXr-  [        U5      U-  -   nX-  [        U	5      U-  -   n[        R
                  " X/SS9n[        R
                  " X/SS9nX4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr   r   r   )	unsqueezeshaperepeat_interleaver&   r    cat)qkcossinposition_idsunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds                r   apply_rotary_pos_embr9   1   s6   ( --
&C
--
&C c'SYYr]a'''
(
:
:1"
:
EC
c'SYYr]a'''
(
:
:1"
:
EC 2Jc;J;&'3
+;)<6c;J;&'3
+;)<6 {{51C78G{{51C78G ii)r2Gii)r2Gr   c                   >   ^  \ rS rSrSS\S\\   4U 4S jjjrSrU =r	$ )GlmAttention[   config	layer_idxc                    > [         TU ]  X5        [        R                  " UR                  U R
                  -  UR                  SS9U l        g )NF)bias)super__init__nnLinearnum_attention_headshead_dimhidden_sizeo_proj)selfr=   r>   	__class__s      r   rB   GlmAttention.__init__\   s:    +ii : :T]] JFL^L^ejkr   )rH   )N)
r   r   r   r   r   r   intrB   r   __classcell__)rJ   s   @r   r;   r;   [   s#    ly lXc] l lr   r;   c                       \ rS rSrSrg)GlmForCausalLMa   r   Nr   r   r   r   rO   rO   a   r   r   rO   c                       \ rS rSrSrg)GlmForSequenceClassificatione   r   Nr   r   r   r   rR   rR   e   r   r   rR   c                       \ rS rSrSrg)GlmForTokenClassificationi   r   Nr   r   r   r   rU   rU   i   r   r   rU   )GlmPreTrainedModelGlmModelrO   rR   rU   )Nr   )typingr   r    torch.nnrC   torch.utils.checkpointutilsr   llama.modeling_llamar   r   r	   r
   phi3.modeling_phi3r   configuration_glmr   
get_loggerr   logger_CHECKPOINT_FOR_DOCr   r&   r9   r;   rO   rR   rU   __all__r   r   r   <module>rd      s           ) ( 
		H	%& 	W 	6'Tl> l	% 		#A 		 ; 	r   