
    eTh                     &   S SK JrJr  S SKrSSKJrJr  \" 5       r    SS\R                  R                  S\R                  S\R                  S\R                  S	\\R                     S
\S\\   S\\   S\\   S\\R                  S4   4S jjrg)    )OptionalTupleN   )_flash_attention_forward!flash_attn_supports_top_left_maskmodulequerykeyvalueattention_maskdropoutscalingsliding_windowsoftcapreturnc	                 p   UR                   S   n
UR                  SS5      nUR                  SS5      nUR                  SS5      nS nUR                  [        R                  :X  a  [        R
                  " 5       (       a  [        R                  " 5       nOf[        U R                  S5      (       a  U R                  R                  nO4[        S U R                  5        5       5      R                  R                  nU	R                  SS 5        [        UUUU4U
U R                  UUUU[         US.U	D6nUS 4$ )Nr      _pre_quantization_dtypec              3   |   #    U  H2  n[        U[        R                  R                  5      (       d  M.  Uv   M4     g 7f)N)
isinstancetorchnnLinear).0layers     a/var/www/auris/envauris/lib/python3.13/site-packages/transformers/integrations/flash_attention.py	<genexpr>*flash_attention_forward.<locals>.<genexpr>,   s+     j3C%zRWY^YaYaYhYhGi3Cs   -<	<	is_causal)query_lengthr   r   softmax_scaler   r   use_top_left_masktarget_dtype)shape	transposedtyper   float32is_autocast_enabledget_autocast_gpu_dtypehasattrconfigr   nextmodulesweightpopr   r   _use_top_left_mask)r   r	   r
   r   r   r   r   r   r   kwargsseq_lenr#   attn_outputs                r   flash_attention_forwardr4      s    kk!nG OOAq!E
--1
COOAq!E L{{emm#$$&& 779LV]]$=>>!==@@Lj6>>3CjjqqwwL JJ{D!*	
 ""%,! K      )g        NNN)typingr   r   r   modeling_flash_attention_utilsr   r   r0   r   ModuleTensorfloatintr4    r5   r   <module>r=      s    "  h 78  #$(#6HHOO6<<6 
6 <<	6
 U\\*6 6 e_6 SM6 e_6 5<<6r5   