
    [Th&V                      % S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
JrJrJr  S SKJrJr  S SKJr  S SKJr  S SK	JrJrJrJrJrJrJrJrJr  S SKJrJrJr  S S	K J!r!  S SK"r"S S
K"J#r#J$r$J%r%  S SK&J's  J(s  J)r*  S SK+J,s  J-r.  S SK/r0S SK1r0S SK2J-s  J3r4  S SK5J6r6  S SK7J8r8  S SK9J:r:  S SK;J<r<  S SK=J>r>J?r?J@r@JArAJBrB  S SKCJDrD  S SKEJFrFJGrGJHrHJIrIJJrJJKrK  S SKLJMrM  S SKNJOrOJPrPJQrQ  S SKRJSrS  SSKTJUrUJVrV  SSKWJXrXJYrYJZrZ  SSKVJ[r[J\r\J]r]J^r^J_r_  SSK`Jara  SSKbJcrcJdrdJereJfrf  SSKgJhrh  SSKiJjrjJkrk  SSK-JlrlJmrmJnrnJoroJprpJqrqJrrrJsrsJtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{  SSK|J}r}J~r~Jr  \(       a  S SKJr  SS KJr  SS!KJr  SS"K-Jr  O\rS#\S$'    S SKr\GR                  rS%r\" S'5      r\" S(5      r\" S)5      r\\\#4   rS#\S*'   \\\\#4   rS#\S+'   \GR*                  " \5      r\R$                  " \GR0                  S,S-9r\0R                  GR2                  r \\S.\\S.4   S/S0\\\\\\S.4   S.S/S04         4   rS#\S1'   \GR:                  " S%S29 " S3 S45      5       rGSS5 jrGSS6 jrGS	S7 jrGS	S8 jr      GS
S9 jr/ S:Qr/ S;Qr GS     GSS< jjrGSS= jr GS     GSS> jjr\GSGSS? jj5       r\GSGSS@ jj5       r GS     GSSA jjr    GSSB jr    GSSC jrGSSD jrGSSE jrGSSF jr        GSSG jr      GSSH jr " SI S05      r\t" S&S29 " SJ SK5      5       r\t " SL SM\5      5       rGSSN jr\t " SO SP\5      5       r\t " SQ SR\5      5       r\" SS5      \" ST5      \" SU5      \" SV5      \" SW5      \" SX5      SY.rSZ\S['    GS       GSS\ jjr\t " S] S^\5      5       r\\\#   \\#   /\~4   r " S_ S`\5      r " Sa Sb\5      r " Sc Sd\5      r\t " Se Sf\5      5       r\t " Sg Sh\5      5       r\t " Si Sj\5      5       rGSSk jrGSSl jr     GS             GSSm jjr      GSSn jr\t " So Sp\5      5       r\t " Sq Sr\5      5       r\t " Ss St\5      5       r\t " Su Sv\5      5       r\t " Sw Sx\5      5       r\t " Sy Sz\5      5       r\t " S{ S|\5      5       r\t " S} S~\5      5       r " S S\5      r\t " S S\5      5       r\t " S S\5      5       r\t " S S\5      5       r      GSS jrGSS jr " S S5      r\t " S S\5      5       r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r\t " S S\5      5       r " S S\5      r\t" S&S29 " S S\5      5       r\t" S&S29 " S S\\5      5       r " S S\5      r " S S\5      r " S S\5      r\t " S S\5      5       r\t " S S\5      5       r\t" S&S29 " S S\5      5       r " S S\5      r " S S\5      r\\\\\\\\\\\4      4   r " S S5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r\t" S&S29 " S S\5      5       r " S S\5      r " S S\5      r\t" S&S29 " S S\5      5       r\t" S&S29 " S S\5      5       r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      Gr  " S S\5      Gr\t" S&S29 " S S5      5       Gr " S S\5      Gr\t" S&S29 " S SG\5      5       Gr\t " S S\5      5       Gr " S S\5      Gr\GR:                   " S S\5      5       Gr " S S.G\5      Gr " S SG\5      Gr	\t" S&S29 " S S\5      5       Gr
GS S jGr\t" S&S29 " S S\5      5       Gr\t" S&S29 " S S\5      5       Gr    GS!S jGr\t" S&S29 " S S\5      5       Gr " S SG\5      Gr " S S\5      Gr\t " S SG\5      5       Gr\t " GS  GSG\5      5       Gr " GS GSG\5      Gr " GS GSG\5      GrGS"GS jGrg! \ a    SrS&r GN$f = f(#      )annotationsN)	GeneratorIterableSequence)AbstractContextManagernullcontext)Enum)partial)	AnyCallableClassVarLiteralOptionaloverloadTYPE_CHECKINGTypeVarUnion)assert_neverNever	TypeAlias)patch)ExprIntegerSymbol)identity)GraphModuleSerializer)can_auto_functionalize)metrics)compute_required_storage_lengthis_boolean_dtypeis_float_dtypemake_channels_last_strides_for
StrideType)get_schema_info)compute_unbacked_bindingsfree_unbacked_symbolsrebind_unbackedresolve_unbacked_bindingsShapeEnvSymTypes
OrderedSet)CleanDivFloorDivModularIndexing)SymT   )configdependencies)BackendFeatureget_scheduling_for_deviceindex_prevent_reordering)Depextract_free_unbacked_symbols#extract_input_node_reduction_rangesextract_read_writesvar_builder)LoopBody)OpCounterCSEOpCountResultReductionType	StoreMode)benchmarker)DevicePropertiesReductionHint)argsortargsort_symcache_on_selfceildivconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warningget_kernel_metadatair_dataclass
is_dynamicis_gpu	sympy_dotsympy_index_symbolsympy_index_symbol_with_prefixsympy_product
sympy_subs)opsOpsValueV)Node)CUDATemplate)GraphLowering)IndentedBufferr   rX   TF_T_U_V_IntLike_NumLikez  prefix	TensorBoxr   IRNode_NodeOrNodes)frozenc                  >    \ rS rSr% S\S'   S\S'   S\S'   S\S	'   S
rg)GraphPartitionSignature   z5dict[str, Union[IRNode, sympy.Expr, TorchBindObject]]input_nodeslist[IRNode]output_nodeszdict[str, bool]input_deallocationboolskip_cudagraph N__name__
__module____qualname____firstlineno____annotations____static_attributes__ro       J/var/www/auris/envauris/lib/python3.13/site-packages/torch/_inductor/ir.pyrg   rg      s     GF ('rw   rg   c                &   ^ SU4S jjmT" U 5        g )Nc                  > U c  g [        U [        [        45      (       a  U  H  nT" U5        M     g [        U [        5      (       a   U R	                  5        H  nT" U5        M     g [        U [
        [        [        [        [        R                  R                  R                  [        [        [        [         4	5      (       d   S[#        U 5       S35       eg )NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])
isinstancelisttupledictvalues
ExpandViewDynamicScalarAssertScalarrb   sympylogicboolalgBooleanr   intEffectfulKernelShapeAsConstantBuffertype)nodesnode_check_tensorboxs     rx   r   %validate_ir.<locals>._check_tensorbox   s     =e}-- & t$$ & ' ! KK''//#)
   e%jk rw   )r   Optional[_NodeOrNodes]returnNonero   )node_or_nodesr   s    @rx   validate_irr      s    < ]#rw   c                F   ^  [        T [        5      (       d   eSU 4S jjnU$ )Nc                 0   > [        [        T5      " U 0 UD6$ N)getattrrT   )argskwargsnames     rx   fnops_wrapper.<locals>.fn   s    sD!42622rw   )r   objectr   r   r   rU   )r{   str)r   r   s   ` rx   ops_wrapperr      s"    dC    3 Irw   c           
     f   ^ [        [        U [        [        U 5      5      5      5      mSU4S jjnU$ )Nc                   > [        U 5      [        T5      :X  d   e[        [        U 5      5       Vs/ s H
  oTU      PM     sn$ s  snf r   lenrange)indexi	inv_orders     rx   reindex inverse_reorder.<locals>.reindex   sC    5zS^+++-23u:->?->il#->???   Ar   Sequence[_T]r   r   )r~   zipr   r   )orderr   r   s     @rx   inverse_reorderr      s*    Sc%j 123I@ Nrw   c                   ^  SU 4S jjnU$ )Nc                   > [        U 5      [        T5      :X  d   e[        [        U 5      5       Vs/ s H
  oTU      PM     sn$ s  snf r   r   )r   r   r   s     rx   r   same_reorder.<locals>.reindex   sB    5zSZ''').s5z):;):AeAh):;;;r   r   ro   )r   r   s   ` rx   same_reorderr      s    < Nrw   c                   ^ ^ SU U4S jjnU$ )Nc                    > T" T" U 5      5      $ r   ro   )r   reindex1reindex2s    rx   r    fuse_reindexing.<locals>.reindex  s    ((rw   )r   r   r   zSequence[_V]ro   )r   r   r   s   `` rx   fuse_reindexingr     s    ) ) Nrw   )   r      r1   )   r   r   r   r1   c                <    Uc  [        U 5      nU$ [        X5      nU$ )z)
Convert strides to fill order (argsort)
)rD   rE   )seq	shape_env
sorted_idxs      rx   get_fill_orderr     s-     $+CL
  !0
rw   c                    [        U 5       VVs0 s H  u  pX!_M	     nnn[        [        U 5      5       Vs/ s H  oCU   PM	     nnU$ s  snnf s  snf )zx
Convert stride order to fill order
For channel last format,

stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0]
)	enumerater   r   )r   idxposlookupr   
fill_orders         rx   stride_order2fill_orderr     sR     (1'78'783ch'7F8%*3u:%67%6)%6J7 97s
   AAc                    [        X5      n[        [        U 5      5       Vs/ s H  nSPM     nn[        U5       H	  u  pVXTU'   M     U$ s  snf )z!
Convert strides to stride order
r   )r   r   r   r   )r   r   r   _outr   elems          rx   get_stride_orderr   *  sL     !/s >JCHo
&o1oC
&Z(D	 )J 's   A
c                    g r   ro   xguard_shapes     rx   ir_node_to_tensorr   7  s    KNrw   c                    g r   ro   r   s     rx   r   r   ;  s    LOrw   c                   U c  g U(       d%  [         R                  R                  R                  nO[        nU R                  5        Vs/ s H
  o2" U5      PM     nn[        U 5      (       a0  U R                  5       R                   Vs/ s H
  o2" U5      PM     nnO[        R                  U5      nU R                  5       nU R                  5       n[        U5      n[        U5      n[         R                  R                  R                  R                  5          [         R"                  " XEXgS9R%                  5       nS S S 5        U$ s  snf s  snf ! , (       d  f       W$ = f)N)sizestridedtypedevice)rV   graphsizevars	size_hintr   get_sizeis_storage_and_layout
get_layoutr   FlexibleLayoutcontiguous_strides	get_dtype
get_devicerI   r   suppress_guardstorchempty_stridedzero_)	r   r   shape_fnsr   r   r   r   ts	            rx   r   r   ?  s    	y 77##--!".AHQKD.Q'(||~'<'<='<!(1+'<=2248KKME\\^F"4(D$V,F	
			#	#	3	3	5E

%' 	
 
6 H / > 
6	5 Hs   	EE$E
E c                D    [        U [        5      (       a
  U (       d  S /$ U $ r   )r{   r|   values    rx   may_convert_to_optionalr   [  s!     %u vLrw   c                @   [        U [        5      (       d  U c  U $ [        U [        R                  5      (       a  U R                  $ [        U [
        [        45      (       a  [        U R                  5       5      $ [        SU  S[	        U 5      R                   S35        g )Nzget_device_type(: ))r{   r   r   r   r   rc   
OutputSpecget_device_typer   r   rq   r   s    rx   r   r   e  sz     !SQY	Au||	$	$vv	A
+	,	,q||~..#A3ba)9)9(:!<=rw   c                    [        U 5      nUS;   a  [        [        U S35      S:X  a  ggUb  [        U5      =nc  gSSKJn  [        U[        5      (       d   e[        X#5      $ )N)cpucuda_backendtritonTFr1   )TritonScheduling)	r   r   r2   r5   codegen.tritonr   r{   r   
issubclass)r   r   device_schedulingr   s       rx   	is_tritonr   q  sn    QF  6fXX./8;!:6!BBK0'....'::rw   c                    [        U 5      S:H  $ )Nr   )r   r   s    rx   is_cpur     s    1&&rw   c           	       ^ ^ [        T [        5      (       a  T R                  5       c  g[        UU 4S j[	        [        T R                  5       5      S-
  5       5       5      n[        R                  R                  R                  T R                  5       S   5      S:H  =(       d=    [        R                  R                  R                  T R                  5       S   5      S:*  nU=(       a    U$ )NFc              3     >#    U  HE  n[         R                  R                  R                  TR	                  5       U   5      T-  S :H  v   MG     g7f)r   N)rV   r   r   r   
get_stride).0r   	alignmentr   s     rx   	<genexpr>-is_aligned_realized_tensor.<locals>.<genexpr>  sB      /A 
			#	#ALLN1$5	6	BqH/s   AAr1   )r{   rc   maybe_get_strideallr   r   r  rV   r   r   r   r   )r   r  aligned_stridesaligned_last_dims   ``  rx   is_aligned_realized_tensorr    s    a  A$6$6$8$@ s1<<>*Q./ O 	
""1<<>"#56!; 	=77%%ajjl2&671<  //rw   c                   [        U5      [        U 5      :X  a  [        U 5      [        U5      :X  d   e[        X U5       H  u  p4n[        R                  R                  R                  US5      (       a  M7  [        R                  R                  R                  XE5      (       a  Mg  [        R                  R                  R                  U5      [        R                  R                  R                  U5      :X  a  M    g   g)zH
Returns true if the strides are equal, ignoring dimensions of size 1 .
r1   FT)r   r   rV   r   r   statically_known_leqstatically_known_equalssymbolic_hint)strides1strides2shapedims1s2s         rx   significant_strides_equalr    s     u:X&3x=CM+III5H57700a88ww77
 
''""0048H8H8V8V9
 
  6 rw   c                ^   [        U 5      (       d  U $ [        S [        XR                  5       5       5       5      (       a  U $ [	        XR                  5       U R                  5       5      (       d  U $ [        U 5      u  p#/ UR                  Qn[        U R                  5       5       H<  u  pV[        R                  R                  R                  US5      (       d  M6  X   XE'   M>     [        UR                  UR                  UR                   UUR"                  5      n[%        ['        X'S95      $ )a  
Tries to match the strides of the tensor to those in the meta_strides. Strides of insignificant
dimensions - size 0 or 1 - will be updated.

If there are real stride differences (NHWC vs NCHW), or the tensor is not realized, then the input will be returned
c              3  x   #    U  H0  u  p[         R                  R                  R                  X5      v   M2     g 7fr   rV   r   r   r  r  r  r  s      rx   r  2try_match_insignificant_strides.<locals>.<genexpr>  s1      7FB 	
00887   8:r1   datalayout)r   r	  r   r  r  r   as_storage_and_layoutr   r   rV   r   r   r  FixedLayoutr   r   r   offsetrb   ReinterpretView)tensorstridesstorage
old_layout
new_strider   r   
new_layouts           rx   try_match_insignificant_stridesr+    s    !((
 '#4#4#67   $W.?.?.A6??CTUU/7G%:$$%J&//+,7700A66#JJM - J _'EFFrw   c                  b   \ rS rSr% \" 5       rS\S'   \R                  " SS9r	S\S'   \R                  " SS9r
S\S	'   \R                  " SS9rS
\S'   \\R                  SES j5       5       rSFS jrSGS jrSHS jrSIS jrSJS jrSKS jrSLSMS jjr SN       SOS jjrSPS jrSQS jrSRS jrSSS jrSTS jrSUS jrSVS jrSWS jrSXS jr \!SYS j5       r"SZS jr#SVS  jr$S[S! jr%S\S]S# jjr&S^S$ jr'S_S% jr(SVS& jr)S`S' jr*SaS( jr+SbS) jr,SXS* jr-ScS+ jr.S[S, jr/S\SdS- jjr0SeS. jr1SGS/ jr2SfS0 jr3SGS1 jr4 Sg     ShS2 jjr5SiS3 jr6SjS4 jr7 Sg     SkS5 jjr8SlS6 jr9SmS7 jr:SnS8 jr;SoS9 jr<SpS: jr=S[S; jr>SqS< jr?SVS= jr@SVS> jrASrS? jrBSsS@ jrCScSA jrDSsSB jrE\F(       a  \!SPSC j5       rGSDrHg"SDrHg")trc   i  zClassVar[OrderedSet[Any]]_current_originsF)initOrderedSet[Any]originsOptional[list[str]]	tracebackOptional[torch.fx.Node]origin_nodec              #     #    [         R                  nX-  [         l         S v   U[         l        g ! U[         l        f = f7fr   )rc   r-  )r0  olds     rx   current_originsIRNode.current_origins  s4      %%"%-	*&)F#cF#s   A1 A>Ac                0    [         R                  XU5        g r   )r   __setattr__)selfattrr   s      rx   _post_init_setattrIRNode._post_init_setattr  s     	4u-rw   c                    U R                  S[        U R                  5      5        U R                  S[        R                  (       a  [
        R                  " 5       OS 5        U R                  SS 5        g )Nr0  r2  r4  )r=  r,   r-  r2   debug_ir_tracebackr2  format_stackr;  s    rx   __post_init__IRNode.__post_init__  sV    	:d6K6K+LMV5N5N//1TX	
 	t4rw   c                B    [        S U R                  5        5       5      $ )Nc              3  8   #    U  H  oR                   v   M     g 7fr   r   r  deps     rx   r  (IRNode.get_read_names.<locals>.<genexpr>       ?.>s((.>   r,   	get_readsrB  s    rx   get_read_namesIRNode.get_read_names      ?dnn.>???rw   c                    U R                   $ r   )r2  rB  s    rx   get_tracebackIRNode.get_traceback  s    ~~rw   c                    U R                   $ r   r4  rB  s    rx   get_origin_nodeIRNode.get_origin_node      rw   c                    g r   ro   rB  s    rx   get_defining_opIRNode.get_defining_op      rw   c                d    S[        U SS5       3nU(       a  [        U5      S:  a  US S  S3nU/$ )Nzorigins=r0   @   =   z...)r   r   )r;  shortenr0  s      rx   common_reprIRNode.common_repr  s@    WT9b9:;s7|b( "c*Gyrw   c                .   [        U5      [        U R                  U5      5      -   n[        [        [        U5      5      nU(       a5  [	        SR                  U5      5      n[        U 5      R                   SU S3$ [        U 5      R                   SU S3$ )Nz,
z(
z
)(r   )r|   rc  mapr   indentjoinr   rq   )r;  linesrb  	multiline	new_liness        rx   
str_helperIRNode.str_helper  s     Ud4#3#3G#<==Se_%uzz%01I4j))*#i[<<4j))*!E7!44rw   c                    U R                   $ r   r   rB  s    rx   r   IRNode.get_dtype      zzrw   c                D     U R                  5       $ ! [         a     g f = fr   )r   NotImplementedErrorrB  s    rx   maybe_get_dtypeIRNode.maybe_get_dtype  s&    	>>##" 		    
c                2    [        S[        U 5       S35      e)Nz#get_layout() is not implemented by !rt  r   rB  s    rx   r   IRNode.get_layout  s    !$GT
|ST"UVVrw   c                D     U R                  5       $ ! [         a     g f = fr   )r   rt  rB  s    rx   maybe_get_layoutIRNode.maybe_get_layout  &    	??$$" 		rw  c                "    U R                  5       $ r   )r   rB  s    rx   get_output_specIRNode.get_output_spec%  s      rw   c                D     U R                  5       $ ! [         a     g f = fr   )r  rt  rB  s    rx   maybe_get_output_specIRNode.maybe_get_output_spec(  s(    	''))" 		rw  c                >    [        U R                  5       [        5      $ )z4True for single tensor output (excludes MultiOutput))r{   r  LayoutrB  s    rx   has_tensor_outputIRNode.has_tensor_output.  s    $446??rw   c                2    [        S[        U 5       S35      e)Nz!get_size() is not implemented by ry  rz  rB  s    rx   r   IRNode.get_size2  s    !$Ed4j\QR"STTrw   c                D     U R                  5       $ ! [         a     g f = fr   )r   rt  rB  s    rx   maybe_get_sizeIRNode.maybe_get_size5  %    	==?"" 		rw  c                "    U R                  5       $ r   r   rB  s    rx   r  IRNode.shape;  s    }}rw   c                4    [        U R                  5       5      $ r   )rR   r   rB  s    rx   	get_numelIRNode.get_numel?  s    T]]_--rw   c                    [         R                  R                  R                  [        R
                  " U R                  5       S5      5      $ Nr   rV   r   r   is_expr_static_and_truer   Eqr  rB  s    rx   is_zero_elementsIRNode.is_zero_elementsB  0    ww77AQST8UVVrw   c                0    [        S[        U 5       35      e)a  
If the IRNode refers to data which has not been materialized (e.g.,
it is a Pointwise/Reduction that could potentially have more
compute fused into it), realize the IRNode into physical memory,
ending the possibility of fusing into it, but allowing, e.g., multiple
users to access the data without having to recompute.

Check StorageBox.realize for a particularly notable implementation.

TODO(ezyang): I think, in principle, every IRNode should have an
implementation of this, and most of the time no-op is OK, but you
really do have to audit each IRNode for this, so for now, raise
an error if it's not implemented.  Note that some code in graph.py
will catch this thrown error and suppress it with a warning.
zrealize NYI on rz  rB  s    rx   realizeIRNode.realizeE  s      "ODJ<"@AArw   Nc                0    [        S[        U 5       35      e)Nzcodegen_reference NYI on rz  r;  writers     rx   codegen_referenceIRNode.codegen_referenceW  s    !$=d4j\"JKKrw   c                    g r   ro   rB  s    rx   r   IRNode.get_deviceZ  r]  rw   c                0    U R                  5       nUc   eU$ r   )r   r;  r   s     rx   get_device_or_errorIRNode.get_device_or_error]  s    "!!!rw   c                    gNFro   rB  s    rx   has_exceeded_max_readsIRNode.has_exceeded_max_readsb      rw   c                >    [        [        U 5      R                  5      er   rt  r   rq   rB  s    rx   make_loaderIRNode.make_loadere      !$t*"5"566rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   make_indexerIRNode.make_indexerh  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   r  IRNode.get_stridek  r  rw   c                D     U R                  5       $ ! [         a     g f = fr   )r  rt  rB  s    rx   r  IRNode.maybe_get_striden  r  rw  c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_nameIRNode.get_namet  r  rw   c                D     U R                  5       $ ! [         a     g f = fr   )r  rt  rB  s    rx   maybe_get_nameIRNode.maybe_get_namew  r  rw  c                    gr  ro   r;  	thresholds     rx   has_large_inner_fnIRNode.has_large_inner_fn}  r  rw   c                    g r   ro   r;  userss     rx   
mark_reuseIRNode.mark_reuse      rw   c                    g r   ro   rB  s    rx   realize_hintIRNode.realize_hint  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   unwrap_viewIRNode.unwrap_view  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   freeze_layoutIRNode.freeze_layout  r  rw   c                >    [        [        U 5      R                  5      er   r  r;  r   allow_paddings      rx   freeze_layout_with_stride_order&IRNode.freeze_layout_with_stride_order       "$t*"5"566rw   c                >    [        [        U 5      R                  5      er   r  r;  r   s     rx   freeze_layout_with_fill_order$IRNode.freeze_layout_with_fill_order  r  rw   c                >    [        [        U 5      R                  5      er   r  r;  r   s     rx   freeze_layout_with_same_order$IRNode.freeze_layout_with_same_order  r  rw   c                >    [        [        U 5      R                  5      er   r  r;  exact_stridesr  s      rx    freeze_layout_with_exact_strides'IRNode.freeze_layout_with_exact_strides  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_read_writesIRNode.get_read_writes  r  rw   c                6    U R                  5       R                  $ r   r  readsrB  s    rx   rN  IRNode.get_reads      ##%+++rw   c                4    [        U R                  5       5      $ r   )r   rN  rB  s    rx   	num_readsIRNode.num_reads  s    4>>#$$rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_storage_numelIRNode.get_storage_numel  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_unbacked_symbol_usesIRNode.get_unbacked_symbol_uses  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_reduction_typeIRNode.get_reduction_type  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_reduction_sizeIRNode.get_reduction_size  r  rw   c                    gr  ro   rB  s    rx   	is_externIRNode.is_extern  r  rw   c                    gr  ro   rB  s    rx   is_no_opIRNode.is_no_op  r  rw   c                >    [        [        U 5      R                  5      er   r  r  s     rx   constant_to_deviceIRNode.constant_to_device  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_mutation_namesIRNode.get_mutation_names  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_operation_nameIRNode.get_operation_name  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   get_inputs_that_alias_output#IRNode.get_inputs_that_alias_output  r  rw   c                    g r   ro   rB  s    rx   r   IRNode.dtype  s    (+rw   ro   )r0  zOrderedSet[Node]r   zGenerator[None, None, None])r<  r   r   r   r   r   r   r   r   OrderedSet[str])r   r1  r   r3  r   zOptional[Operation]T)rb  rm   r   Sequence[str])TT)rj  zSequence[object]rb  rm   rk  rm   r   r   r   torch.dtype)r   zOptional[torch.dtype]r   r  )r   zOptional[Layout]r   r   )r   zOptional[OutputSpec]r   rm   r   Sequence[Expr])r   Optional[Sequence[_IntLike]])r   z.Union[_IntLike, sympy.Rel, Sequence[_IntLike]]r   r   r   Optional[str]r   r  zOptional[IndentedBuffer]r   r   r   Optional[torch.device]r   torch.devicer   $Callable[[Sequence[Expr]], OpsValue]r    Callable[[Sequence[Expr]], Expr]r   Sequence[_IntLike]r   r   r  Optional[int]r   rm   r  r   r   r   r   rc   Fr   	list[int]r  rm   r   r   r   r9  r   r   r   list[_IntLike]r   r   r  r<  r  rm   r   r   r   zdependencies.ReadWritesr   zOrderedSet[Dep]r   r   r   r^   r   OrderedSet[Symbol]r   Sequence[sympy.Expr]r   r+  r   rc   r   r  )Irq   rr   rs   rt   r,   r-  ru   dataclassesfieldr0  r2  r4  staticmethod
contextlibcontextmanagerr7  r=  rC  rO  rS  rW  r[  rc  rm  r   ru  r   r}  r  r  r  r   r  propertyr  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rN  r  r  r  r  r  r  r  r  r  r  r  r   r   rv   ro   rw   rx   rc   rc     s   2<,/>  +00e<G_<%0%6%6E%BI"B+6+<+<%+HK(H*  *.5@  PT	5%	504	5HL	5		5W!@U  .WB$L
777777 7<77/37	7
77 DI7+7<@7	7
7,%77777777 	+ 
+ rw   c                      \ rS rSrSS jrSS jrSS jrSS jrSS jrSS jr	SS jr
SS	 jrSS
 jrSS jrSS jrSS jrSS jrSS jrSS jrSrg) 	Operationi  c                    S U l         g r   operation_namerB  s    rx   rC  Operation.__post_init__  s
    -1rw   c                    [         er   rt  rB  s    rx   r   Operation.get_device      !!rw   c                @    [        U S5      (       d   eU R                  $ Nr4  )hasattrr4  rB  s    rx   rW  Operation.get_origin_node  s!    t]++++rw   c                @    [        U S5      (       d   eU R                  $ )Nr0  )rZ  r0  rB  s    rx   get_originsOperation.get_origins  s    tY''''||rw   c                8    U R                   c   eU R                   $ r   rQ  rB  s    rx   r  Operation.get_operation_name  s     ""..."""rw   c                    gr  ro   rB  s    rx   r  Operation.is_extern  r  rw   c                    gr  ro   rB  s    rx   r  Operation.is_no_op  r  rw   c                    [         er   rU  rB  s    rx   r  Operation.get_read_writes  rW  rw   c                &    XR                  5       ;   $ r   )rO  )r;  r   s     rx   
is_user_ofOperation.is_user_of  s    **,,,rw   c                B    [        S U R                  5        5       5      $ )Nc              3  8   #    U  H  oR                   v   M     g 7fr   rG  rH  s     rx   r  +Operation.get_read_names.<locals>.<genexpr>  rK  rL  rM  rB  s    rx   rO  Operation.get_read_names  rQ  rw   c                6    U R                  5       R                  $ r   r  rB  s    rx   rN  Operation.get_reads  r  rw   c                    [         er   rU  rB  s    rx   get_outputsOperation.get_outputs  rW  rw   c                    [        5       $ r   r+   rB  s    rx   get_unbacked_symbol_defs"Operation.get_unbacked_symbol_defs  
    |rw   c                    [        5       $ )a}  
Returns the unbacked symbols which are required to be in scope in
order to successfully perform codegen for this buffer.  For example,
a buffer that corresponds to an extern kernel call that takes i0 as
an argument would return {i0} here.  This is used to generate necessary
dependencies that ensure we actually bind i0 in codegen before you
try to use it.

Note that this is NOT transitive; in particular, if this buffer takes
in as input another buffer with dynamic shape (e.g., (i0,)), we will
not report it here, because you will already have a dependency
on that buffer, which will eventually have a dependency on i0 if
necessary.
r+   rB  s    rx   r  "Operation.get_unbacked_symbol_uses  s     |rw   c                    g)z
Gets extra global memory size needed by this buffer.
Some algorithms (e.g. group gemm) may require extra global memory in the generated code.
r   ro   rB  s    rx   get_workspace_sizeOperation.get_workspace_size  s    
 rw   rQ  Nr  r(  r  )r   r/  r2  r   r>  )r   r   r   rm   r  r?  r   list[Buffer]r   zOrderedSet[sympy.Symbol]r@  )rq   rr   rs   rt   rC  r   rW  r]  r  r  r  r  rh  rO  rN  rq  rt  r  rz  rv   ro   rw   rx   rO  rO    sN    2" #"-@,""rw   rO  c                  h  ^  \ rS rSr% S\S'   S\S'   S\S'   S\S	'   S S
 jrS!S jrS"U 4S jjrS#S jr\r	S$S jr
S%S jrS&S jrS&S jr\S'S j5       r\\R$                  4S(S jj5       r\S)S j5       rS*S jr\S#S j5       rS+S,S jjrS S jrS-S jrS.S jrS/S jrS0S jrS1S jrS2S jrSr U =r!$ )3Loopsi  r+  r   r  r   Callable[..., Any]inner_fnr1  rangesc                x    [        5       R                  " / S U R                   5       QU R                  5       P76 $ )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr   r&   r  es     rx   r  1Loops.get_unbacked_symbol_uses.<locals>.<genexpr>  s     <1#A&&rL  )r,   unionr  inner_fn_free_unbacked_symbolsrB  s    rx   r  Loops.get_unbacked_symbol_uses  s9    |!! 
<<
//1
 	
rw   c                   U R                  SU R                  R                   S3[        U R                  5      U R                  5       /U Vs/ s H  o" S[        X5       3PM     sn-   SU R                  < 3/-   5      $ s  snf )N'=origin_node=)rm  r   r   r   r   inner_fn_strr   r4  )r;  namesr   s      rx   _to_strLoops._to_str  s    DKK$$%Q'DJJ!!#
 <AA54q,-.5AB d..1234
 	
 Bs   B
c                "   > [         TU ]  5         g r   )superrC  r;  	__class__s    rx   rC  Loops.__post_init__&  s    rw   c                $    U R                  S5      $ )Nr  r  rB  s    rx   __str__Loops.__str__)  s    ||K((rw   c                    U R                   $ r   r   rB  s    rx   r   Loops.get_device.      {{rw   c                    U R                   $ r   rV  rB  s    rx   rW  Loops.get_origin_node1  rY  rw   c                    U R                   $ r   r  rB  s    rx   r   Loops.get_size4  r  rw   c                    U R                   $ r   r  rB  s    rx   get_pointwise_sizeLoops.get_pointwise_size7  r  rw   c                    UR                  SS 5      nUR                  SS 5      nU " U0 UD6nUR                  SU5        UR                  SU=(       d    UR                  5        [        R	                  U5      $ )Nr4  r2  )popr=  r2  rb   create)clsr   r   r4  tbrs         rx   r  Loops.create:  so    jj5ZZT*    	
]K8	["*;<""rw   c                    [        U 5       VVs/ s H0  u  p#US:X  a  [        R                  R                  O
[	        X5      PM2     snn$ s  snnf Nr1   )r   r   SZerorQ   )r  ra   nr   s       rx   _indexLoops._indexH  sI     "&)
) FEGGLL(Fv(QQ)
 	
 
s   7A
c                |   [        [        R                  " 5       5      n[        R                  " U5         [        R
                  " [        SS5         U R                  " U R                  5       6   UR                  5       sS S S 5        sS S S 5        $ ! , (       d  f       O= f S S S 5        g ! , (       d  f       g = fNallow_indexingT)
r=   rV   MockHandlerset_ops_handlerr   r   r   r  inner_fn_argsgetvalue)r;  	opcounters     rx   inner_fn_opcountLoops.inner_fn_opcountO  sy     1	i(LL)94@MM4--/0%%' A@ )(@@@ )((s#   B--B?	B-
B 	B--
B;c                :    U R                  U R                  5      4$ r   )r  r  rB  s    rx   r  Loops.inner_fn_argsY  s    DKK(**rw   c                t    [         R                  R                  " U R                  /U R	                  5       Q76 $ r   )rV   KernelFormatterHandlerir_to_stringr  r  rB  s    rx   r  Loops.inner_fn_str\  s3    ''44MM
 ..0
 	
rw   c                z    Uc  Sn[        U[        R                  5      nU R                  5       R                  U:  $ r  )maxr2   realize_opcount_thresholdr  num_opsr  s     rx   r  Loops.has_large_inner_fnb  s9    I	6#C#CD	$$&..::rw   c                d    U R                  U R                  5      n[        U R                  U5      $ r   )r  r  r8   r  )r;  r   s     rx   r  $Loops.inner_fn_free_unbacked_symbolsh  s%    DKK(,T]]EBBrw   c                   [         R                  " [        SS5         U R                  5       (       aJ  [	        U R                  5       U R                  5       U R                  5       5      R                  sS S S 5        $ [	        U R                  5       U R                  5       5      R                  sS S S 5        $ ! , (       d  f       g = fr  )	r   r   r   r  r:   r  r   r  r  rB  s    rx   rN  Loops.get_readsl  s    \\.*:DA&&((*$$&MMO++- % BA +$$&MMO % BAAs   AB8=1B88
Cc                H    [        U R                  5       R                  5      $ r   )r,   r  read_buffersrB  s    rx   rO  Loops.get_read_namesz  s    $//1>>??rw   c                H    [        U R                  5       R                  5      $ r   )r   r  r  rB  s    rx   r  Loops.num_reads}  s    4((*7788rw   c                2    [        S[        U 5       S35      e)Nz+get_reduction_size() is not implemented by ry  rz  rB  s    rx   r  Loops.get_reduction_size      !9$t*QG
 	
rw   c                2    [        S[        U 5       S35      e)Nz+get_reduction_type() is not implemented by ry  rz  rB  s    rx   r  Loops.get_reduction_type  r  rw   c                2    [        S[        U 5       S35      e)Nz+constant_to_device() is not implemented by ry  rz  r  s     rx   r  Loops.constant_to_device  r  rw   ro   rB  )r  r  r   r   r  r2  r(  r  r!  )r   r   r   r   r   rb   )r  r1  ra   r0   r   r"  )r   r>   r   zSequence[Sequence[_IntLike]]r   r3  r?  r  r@  rD  r%  rF  )"rq   rr   rs   rt   ru   r  r  rC  r  __repr__r   rW  r   r  classmethodr  rJ  r0   INDEXr  rF   r  r  r  r  r  rN  rO  r  r  r  r  rv   __classcell__r  s   @rx   r  r    s      
	
 ) H  # # :>** 
 
 ( (+ 
 

;C@9




 
rw   r  c                   UR                   (       a   [        R                  " [        S5      U5      $ [        R                  " SU5      $ )Nnanr   )is_floating_pointrT   constantfloat)r   r   s     rx   nop_loader_fnr    s1    ||E%L%00||Au%%rw   c                  V    \ rS rSrS	S jrS
S jrSS jr        SS jrSS jrSr	g)	Pointwisei  c                t    U R                  5       (       a  [        [        U R                  S9$ U R                  $ Nrp  )r  r
   r  r   r  rB  s    rx   r  Pointwise.make_loader  s,      ""=

;;}}rw   c                    / $ r   ro   rB  s    rx   r  Pointwise.get_reduction_size  s    	rw   c                    g r   ro   rB  s    rx   r  Pointwise.get_reduction_type  r]  rw   c                |    U R                  5       n[        R                  " U=(       d    SU" U5      U" U5      5      $ Nunnamed)r  rT   storer;  output_nameindexervarsloaders        rx   store_outputPointwise.store_output  s2     !!#yy1	74=&,OOrw   c                    U R                  5       n[        R                  " [        SU5      " U5      n[	        XR
                  X R                  S9$ FMove this to a given device. Requires that all reads are to constants.override_devicer   r   r  r  )r  r   r   ConstantBufferr  r   r  r;  r   r  s      rx   r  Pointwise.constant_to_device  sB    !!#n.?HPf[[
 	
rw   ro   Nr,  rD  r%  r  r&  r  !Callable[[Sequence[Expr]], Never]r  r"  r   r   rF  )
rq   rr   rs   rt   r  r  r  r  r  rv   ro   rw   rx   r  r    sF    P"P 3P 	P
 
P
rw   r  c                  R    \ rS rSr% S\S'   SrS\S'   S
S jr        SS jrS	rg)Scatteri  r/  output_indexerNr@   scatter_modec           	         U R                  5       n[        R                  " [        SU5      " U5      n[	        UU R
                  UU R                  U R                  U R                  S9$ )r  r   )r   r   r  r  r	  r
  )	r  r   r   r  r  r   r  r	  r
  r  s      rx   r  Scatter.constant_to_device  s[    !!#n.?HP**;;..**
 	
rw   c                    U R                  5       nUc  Sn[        R                  " UU" U R                  U5      5      U" U5      U R                  S9$ )Nr  )mode)r  rT   r  r	  r
  r  s        rx   r  Scatter.store_output  sT     !!##KyyD''-.4L""	
 	
rw   ro   rF  r  )	rq   rr   rs   rt   ru   r
  r  r  rv   ro   rw   rx   r  r    sB    44"L)"

"
 3
 	

 

rw   r  
logical_ormaximumminimummuladdbitwise_xor)anyr  minprodsumxor_sumz"dict[str, Callable[..., OpsValue]]REDUCTION_COMBINE_FNc                   ^ ^^ T [         ;   a	  [         T    $ T S;   a        SUUU 4S jjnU$ T S:X  a        SS jnU$ [        ST  35      e)Nargmaxargminc                  > U u  p#Uu  pETS:X  a  [         R                  " X$5      nO[         R                  " X$5      n[         R                  " X$5      n[	        T5      (       a  [         R
                  " X"5      n[         R
                  " XD5      n	[         R                  " U[         R                  " X5      5      n[         R                  " U[         R                  " X5      5      nT(       a  [         R                  " X55      O[         R                  " X55      n
[         R                  " U[         R                  " Xz5      5      n[         R                  " XbU5      [         R                  " XcU5      4$ )Nr  )	rT   ltgteqr!   ner  logical_andwhere)aba_valuea_indexb_valueb_indexmaskequala_isnanb_isnantiearg_break_ties_leftr   reduction_types              rx   argmax_combine_fn3get_reduction_combine_fn.<locals>.argmax_combine_fn  s     !G G)vvg/vvg/FF7,Ee$$&&2&&2~~dCFF7,DEucoog.OP ' w(VVG- 
 >>$(CDD		$1		$1 rw   welford_combinec                \    U u  p#nUu  pVnXR-
  nXG-   n	Xy-  n
X(U
-  -   X6-   X-  U-  U
-  -   U	4$ r   ro   )r'  r(  a_meana_m2a_weightb_meanb_m2b_weightdelta
new_weight	w2_over_ws              rx   welford_combine_fn4get_reduction_combine_fn.<locals>.welford_combine_fn  s]     &'"F(%&"F(OE!,J -I**emh6BB rw   zunknown reduction_type=)r'  tuple[object, object]r(  rC  r   tuple[OpsValue, OpsValue])r'  #tuple[OpsValue, OpsValue, OpsValue]r(  rE  r   rE  )r  rt  )r3  r   r2  r4  rA  s   ```  rx   get_reduction_combine_fnrF    s     --#N33	/	/	$	)>	&	 	: ! 	,	,	2	2	 1	  "! "$;N;K"LMMrw   c                    ^  \ rS rSr% S\S'   S\S'   S\S'   S\S	'   S S
 jr\rS!U 4S jjrS"S jrS#S jr	          S$S jr
S%S jrS&S jrS!S jrS'S jr\ S(                   S)S jj5       r\          S*S j5       r\\R(                  S4                   S+S jj5       r\      S,S j5       r\      S,S j5       r\        S-S j5       r\              S.S j5       r\            S/S j5       r\                        S0S j5       r\                    S1S j5       r\                      S2S j5       rSrU =r$ )3	Reductioni$  r1  reduction_rangesr?   r3  r  	src_dtyperC   reduction_hintc                $    U R                  S5      $ )N)r  rI  r3  r  rB  s    rx   r  Reduction.__str__,  s    ||LMMrw   c                r   > [         TU ]  5       [        5       R                  " S U R                   5       6 -  $ )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  s     rx   r  5Reduction.get_unbacked_symbol_uses.<locals>.<genexpr>3  s     F0E1#A&&0ErL  )r  r  r,   r  rI  r  s    rx   r  "Reduction.get_unbacked_symbol_uses1  s5    w/1JL4F4FF0E0EF5
 
 	
rw   c                    U R                   $ r   )rI  rB  s    rx   r  Reduction.get_reduction_size6  s    $$$rw   c                    U R                   $ r   )r3  rB  s    rx   r  Reduction.get_reduction_type9      """rw   c           	         [         R                  " U R                  U R                  U R                  U R                  X45      5      n[         R                  " U=(       d    SU" U5      U5      $ r  )rT   	reductionr   rJ  r3  r  store_reduction)r;  r  r  r  reduction_varsr   s         rx   rY  Reduction.store_reduction<  sU     JJNNMM$/	
 "";#;)WT]ERRrw   c                X    [        U R                  5      [        U R                  5      -   $ r   )r   r  rI  rB  s    rx   index_lengthReduction.index_lengthK  s!    4;;#d&;&;"<<<rw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      nX4$ r   )r  r  rI  r0   R0_INDEXr;  r   rindexs      rx   r  Reduction.inner_fn_argsN  s6    DKK(T22DMMBrw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      n[        U R                  X5      $ r   )r  r  rI  r0   r`  r8   r  ra  s      rx   r  (Reduction.inner_fn_free_unbacked_symbolsS  s?    DKK(T22DMMB,T]]EJJrw   c                   U R                  5       n[        R                  " [        SU5      " U5      n[	        UU R
                  UU R                  U R                  U R                  U R                  [        R                  S9$ )r  r   r   r   r  r  rI  r3  rJ  rK  )r  r   r   r  rH  r   r  rI  r3  rJ  rC   DEFAULTr  s      rx   r  Reduction.constant_to_deviceX  sk    !!#n.?HP**;;!22..nn(00	
 		
rw   Nc	                <   SS jn	[         R                  R                  R                  U5      n
[         R                  R                  R                  [	        U5      5      nUS:H  =(       dV    [         R                  R                  U [        R                  5      (       + =(       a    US;  =(       a    [        R                  nU	" U
5      (       a  U	" U5      (       d  [        R                  S4$ [        R                  " U 5      nUR                  nSnU(       a]  [        R                   " [         R"                  R$                  U SS9n[        R                   " [         R"                  R$                  U SS9nO      SS	 jnUnUS:X  a  U" X5      nUS:X  a  [        R&                  U4$ Ub  [)        U[*        5      (       a  [,        R.                  " [0        S
S5         [3        U5      u  nnS S S 5        Wbj  Wbg  [         R                  R                  R                  [	        UU-   5      5      nU
U:X  a,  [4        R7                  SUUUUU5        [        R&                  S4$ [        R&                  U4$ X::  d  XS-  S-  :  a  [        R                  S4$ [9        U UUUUUS:w  a  UOSU[        R                  S9nSS jnU" U5      u  nnU(       a  U" U5      u  nn[;        U5      S:X  a  [        R                  S4$ [<        R>                  " URA                  5       URC                  5       5      u  u  nnnSnSnU H  n[         R                  R                  RE                  UU5      n [         R                  R                  RG                  U U[I        URK                  5       5      5      n![M        S U! 5       5      n"U"(       a  US-  nM  US-  nM     UU:  a  [        R&                  U" X5      4$ [        RN                  U" X5      4$ ! , (       d  f       GN&= f)Nc                .    [        U [        [        45      $ r   )r{   r   r   r   s    rx   
_is_static(Reduction.num_splits.<locals>._is_statics  s    a#w00rw   scanr  r1       T)inner_reductionFc                    gr  ro   )reduction_numel_hint
numel_hints     rx   inner_reduction_splits4Reduction.num_splits.<locals>.inner_reduction_splits  s     rw   r  zUse previous IRNode's range and reduction_ranges instead of split. current ranges: %s, current reduction ranges: %s, current split: %d, new ranges: %s, new reduction ranges: %sr  r   r  rg  c           	     R  ^ [        S [        U R                  5       U R                  5       U R	                  5       S9U S9nUR                  5       nUR                  c   eUR                   V s/ s H=  n [        U [        5      (       d  M  [        U [        R                  5      (       a  M;  U PM?     nn / nSn[        UR                  S S9 H  m[        U4S jU 5       5      (       d  M  UR                  TR                  5        TR                   ["        R$                  R&                  ;   d  Md  ["        R$                  R&                  TR                      n[)        UR*                  SS 5      nUR-                  5         [)        UR*                  SS 5      U:w  d  M  SnM     XE4$ s  sn f )	Nr   r   r   r   r   r  Fc                    U R                   $ r   rG  r   s    rx   <lambda>@Reduction.num_splits.<locals>.get_read_indices.<locals>.<lambda>  s    affrw   keyc              3  T   >#    U  H  oTR                   R                  ;   v   M     g 7fr   )r   free_symbols)r  r  mds     rx   r  AReduction.num_splits.<locals>.get_read_indices.<locals>.<genexpr>  s     F:aBHH111:   %(r   T)ComputedBufferr   r   r   r   r  
range_varsr{   r   r   Numbersortedr  r	  appendr   r   rV   r   name_to_bufferr   r   decide_layout)	r  cbread_writesr  indiceschangedbuforiginal_strider  s	           @rx   get_read_indices.Reduction.num_splits.<locals>.get_read_indices  s]   %<<>++-
 B ,,.K ))555 %///Aa& /9!U\\/J /  
 GG[..4DEF:FFFNN288,ww!''"8"88gg44RWW=*1#**h*M))+"3::x>/Q&*G F ##!s   -F$
F$+F$r   c              3  *   #    U  H	  oS :  v   M     g7fr1   Nro   r  r   s     rx   r  'Reduction.num_splits.<locals>.<genexpr>  s     /w!Aw   )r   r   r   rm   )rr  r   rs  r   r   r   )r  rH  r   ztuple[Sequence[Expr], bool])(rV   r   r   r  rR   has_featurer4   REDUCE_TO_SINGLE_ELEMENTr2   split_reductionsrC   rh  rB   r  multi_processor_count	functoolsr
   choicesreduction_split_factorINNERr{   rb   r   r   r   r9   logdebugrH  r   r3   index_vars_squeezer   r  simplify_with_rangesstride_hintsr|   keysr	  OUTER)#r   	dst_dtyperJ  r  r  rI  r3  reduction_numel
input_noderl  rr  rs  should_splitpropsnum_smmin_elements_per_threadrt  outer_reduction_splitssplit
new_rangesnew_reduction_rangesextracted_numel_hintr  r  r  r  r   rZ  ranges1	num_outer	num_innerr   jr&  outers#                                      rx   
num_splitsReduction.num_splitsg  s   	1  !ww//==oNWW%%33M&4IJ
%/ 
##FN,S,STT (( '' 	 /00Z
5K5K ((!++ ''/,,"$@I@Q@Q		00&$A" AJ@Q@Q		00&%A"
&)  &<" ?*+?LEz$**E11%*Z*K*K\\.2BDI <JG", J
 ).B.N+,77+;+;+I+I%j3G&GH,( ,/CC		G #,!&0	  -22B66 &&-- ;aZ"_, ((!++--;v-E>5(00	
	$B ,A.)!,JGQw<1 ((!++'3'F'FJJL!..0(
$NW 		A  55aAAgg&&33>4#7G /w//EQ	Q	  y  &&(>$)   !&&(>$)  M JIs   P
Pc                R  ^ ^^^^^ T Vs/ s H,  n[         R                  R                  R                  U5      PM.     snm[	        X#5      mSUUU4S jjmUS;   aD  [        SST[        R                  T5      5      R                  5       m      SUU 4S jjmU4S j$ T mT$ s  snf )z1Convert inner_fn from a reduction to an pointwisec                   >^  [         R                  " TU U4S j[        R                  " T Vs/ s H  n[	        U5      PM     sn6  5       5      $ s  snf )Nc              3  6   >#    U  H  nT" TU5      v   M     g 7fr   ro   )r  rb  r   value_fns     rx   r  =Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<genexpr>  s&      # UF++#s   )r  reduce	itertoolsproductr   )r   r   
combine_fnrI  r  s   ` rx   r   *Reduction._unroll_reduction_fn.<locals>.fn  sN    ##"+"3"3,<=,<q%(,<=# 
 >s   Ar  r  Nc                   > U Vs/ s H  n[         R                  " U5      PM     nnT" X5      [        R                  " T" U5      [        R
                  5      4$ s  snf r   )r   expandrT   
index_exprr   int64)r   rb  r   flatten_indexr  s      rx   r  0Reduction._unroll_reduction_fn.<locals>.value_fn-  sO     4::6a%,,q/6:U+NN=#8%++F  ;s    Ac                   > T" U 5      S   $ r  ro   )r   r   s    rx   rz  0Reduction._unroll_reduction_fn.<locals>.<lambda>6  s    E1rw   )r   r1  r   r   )r   r1  rb  r1  r   rD  )	rV   r   r   evaluate_static_shaperF  r"  r   r   r  )	r  rI  r3  rJ  r   r  r  r   r  s	   ``   @@@@rx   _unroll_reduction_fnReduction._unroll_reduction_fn  s     @P
?O!AGG2215?O
 .nH
		 		 11' 112BC	
 ln )3E*  .-HIM
s   3B$c
                `  ^^^^^ [         R                  R                  R                  [	        T5      5      n
U
S:X  an  SU4S jjnU" S5      U" S5      U" S5      U" S5      S.mTTR                  5       ;   d
   T S35       eSUUU4S jjn[        R                  UUU[        U5      S9$ U
S:X  a-  TS;   a	  SU4S	 jjnO	SUU4S
 jjn[        R                  UTXS9$ [        U
[        5      (       a  [         R                  R                  R                  U
5      [        R                  :  aQ  [	        U5      S:w  d  [        UR                  5      (       a(  [        R                  UTU R!                  TTTU5      US9$ U R#                  UTUTUTTU
U	5	      u  pU[$        R&                  :X  a  UnUS:X  a7  U	c   e[)        U	5      u  nnUc   eUc   eU R+                  UTUTUTUUTU5
      $ US:  a  U R-                  UTUTUTTUU5	      $ [.        R                  [1        UTTUTTUUS95      $ )Nr   c                  > T[         R                  :X  a  [        U 5      $ TR                  (       a,  [        U [        R
                  5      (       d   e[        U 5      $ [        U [        R                  5      (       d   e[        U 5      $ r   )	r   rm   r  r{   typingSupportsFloatr  SupportsIntr   )valr  s    rx   py_cnst!Reduction.create.<locals>.py_cnstN  sg    

*9$00%c6+?+?@@@@ :%%c6+=+=>>>>s8Orw   r1   )r  r  r  r  z* not supported for zero-dimension tensors!c                8   > [         R                  " TT   T5      $ r   rT   r  )r   r  r3  rtypes_to_initss    rx   const_fn"Reduction.create.<locals>.const_fnd  s    ||ON$CYOOrw   r  r  c                2   > [         R                  " ST5      $ r  r  )r   r  s    rx   r   Reduction.create.<locals>.fnr  s    <<955rw   c                r   > T Vs/ s H  n[         R                  R                  PM     nnT" X5      $ s  snf r   r   r  r  )r   r   reduction_indexr  rI  s      rx   r   r  w  s2    =M&N=Muww||=MO&N#E;; 'O   $4r  rg  )r  r   r   zUnion[bool, float, int])r   r   r   rU   )rV   r   r   simplifyrR   r  r  r  r|   r{   r   r   r2   unroll_reductions_thresholdrN   r   r  r  rC   rh  r9   !create_multilayer_existing_rangescreate_multilayerrb   rH  )r  r   r  rJ  r  r  rI  r3  rK  r  r  r  r  r   hintr  r  r  r  s     ` ` ``          @rx   r  Reduction.create;  s    ''**33MBR4STa$ qz"1:
qz	O "_%9%9%;; !""LM;P P ##!F|	 $   a!556 6
< < ##Y $  
 00  **?;001v&!+vfkk/B/B ##11.	  $   nn

 ]222!NB;)))/R0,J, )))'33388 $  QY(( 
 
 !!1-#-	
 	
rw   c           	        U S;   aL  [        U5      (       a  [        S5      $ [        U5      (       a  g[        R                  " U5      R
                  $ U S;   aL  [        U5      (       a  [        S5      $ [        U5      (       a  g[        R                  " U5      R                  $ [        U5      (       a  SOSn[        U5      (       a  SOSnUUUUX"U4X"U4[        S5      U4S	.U    $ )
N)r  r  z-infF)r  r  infTr   r1   )r  r  r  r  welford_reducer6  online_softmax_reduce)r!   r  r    r   iinfor  r  )r3  r   zeroones       rx   default_accumulatorReduction.default_accumulator  s     ..e$$V}$!%(({{5)---..e$$U|#!%(({{5)---(//uQ&u--d1#40 $D1&+FmT%:
  	rw   c                :    U S:X  a  g[         R                  X5      $ )Nr  r   )rH  r  r3  r   s     rx   default_valueReduction.default_value  s!     --,,^CCrw   c                    U S:X  a  U$ U S::  a*  US::  a$  U[         R                  :X  a  [         R                  $ U S::  a*  US::  a$  U[         R                  :X  a  [         R                  $ U$ )Nr     i      )rC   r  
OUTER_TINY)r  rs  rK  s      rx   _multilayer_second_step_hint&Reduction._multilayer_second_step_hint  sg     B;!!C<J#-.MDWDW2W +++TMc!-"5"55 +++rw   c                   ^^^^^^	 [         R                  UT/5      m	[        R                  R                  R                  [        R                  " TU-  S5      5      (       + m      SUUUUUU	4S jjnU$ )Nr   c                D  >^^ Uu  nU Gt mnTU-  U-   mSUU	UU4S jjnT
(       at  [         R                  " [         R                  " T[        R                  5      [         R                  " T[        R                  5      5      n[         R
                  " XCT5      $ U" 5       $ )Nc                 $   > T" TT" T /5      5      $ r   ro   )r  r  	new_indexr   s   rx   bodyCReduction._multilayer_wrap_loader.<locals>.wrapper_fn.<locals>.body  s    i');<<rw   )r   rU   )rT   r!  r  r   int32masked)r   r  reduction_blockr  r-  r  r  
block_sizedefaultr  	need_maskr  r   s        @@rx   
wrapper_fn5Reduction._multilayer_wrap_loader.<locals>.wrapper_fn  s     "1_*/'Y ?2_DG= = vvNN7EKK8NN?EKK@ zz$g66vrw   )r   Sequence[Symbol]r  r	  r   rU   )Viewdynamic_reshape_indexerrV   r   r   r  r   r  )
r  r  rI  r  r  r  r  r  r  r   s
    ` ` `` @@rx   _multilayer_wrap_loader!Reduction._multilayer_wrap_loader  sw     ../?/ARS((@@HH_u,a0
 
		#	6F		 	& rw   c                   ^^^ [        S T 5       5      (       d   ST< 35       e[        R                  U[        U5      [        U5      -   5      m      SUUU4S jjnU$ )Nc              3  *   #    U  H	  oS :H  v   M     g7fr  ro   r  r  s     rx   r  DReduction._multilayer_wrap_loader_existing_ranges.<locals>.<genexpr>4  s     3?a6?r  z8Only enabled for numel_hint == 1, found original_ranges=c           	        > U S [        T5       nU [        T5      S  nT" UT" [        U5      [        U5      -   5      5      $ r   )r   r}   )merged_indexnew_reduction_indexoriginal_idxr  r  original_rangesr   s       rx   r  EReduction._multilayer_wrap_loader_existing_ranges.<locals>.wrapper_fn;  sQ     ((>#o*>?L$S%9%;<Ii(51D+EEF rw   )r  rE  r  rE  r   rU   )r	  r
  r  r}   )r  r  r  original_reduction_rangesr  r  r  r   s    ``    @rx   '_multilayer_wrap_loader_existing_ranges1Reduction._multilayer_wrap_loader_existing_ranges+  s     3?333 	
G6HI	
3 ..%uZ'85AU;V'V
		.		!5		 		 		 rw   c                  ^ U[         R                  [         R                  4;  a  UO[         R                  n[        R                  UUUUUUU	U5      nUR                  5         UR                  5       m      SU4S jjn[        R                  R                  R                  [        U5      5      nU R                  XU5      nXWS[        U5       :X  d   e[        R                  [	        UUUUU[        U5      S U	UUS95      $ )I
Break a large reduction up into multiple smaller reductions
recursively
c                   > T" / U QUQ5      $ r   ro   )r   r  intermediate_loaders     rx   intermediate_fn;Reduction.create_multilayer_helper.<locals>.intermediate_fnp  s     ''A'A'ABBrw   Nrg  )r   r1  r  r1  r   rU   )r   float16bfloat16r  rH  r  r  r  rV   r   r   r   rR   r  r   rb   )r  r   r  rJ  r  r  r  r  r  r3  r  rK  intermediate_dtypeintermediater  rs  r  s                   @rx   create_multilayer_helper"Reduction.create_multilayer_helperH  s$   0  ??  	
 !'' 	
 	*668	C%	C8J	C	C
 WW%%//o0NO
99~
 -Cs?/C"DDDD(&!+C,@,B!C-#-	
 	
rw   c
                    [        U5      n
[        XS-
  -   U5      nU R                  Xr5      nU R                  XFXX5      nU R	                  UUUUUU/ UQUPU/UUU	5      $ )r  r1   )rR   r.   r  r  r%  )r  r   r  rJ  r  r  rI  r3  r  rK  r  r  r  r  s                 rx   r  Reduction.create_multilayer  s    $ ((89o;UC
##N>00


 ++feL
 	
rw   c                j    U R                  UUUUU5      nU R                  UUUUUU/ UQUQUU	SU
5      $ )r  r  )r  r%  )r  r   r  rJ  r  r  r  r  r  r3  rK  r  s               rx   r  +Reduction.create_multilayer_existing_ranges  sc    $ @@% 

 ++%+o+
+ 
 	
rw   ro   r2  rB  rD  r%  
r  r&  r  r  r  r"  rZ  r	  r   r   r@  r   zSequence[Sequence[Expr]]rF  r   )r   r+  r  r  rJ  r  r  Callable[..., OpsValue]r  r1  rI  r1  r3  z%Union[ReductionType, Literal['scan']]r  r   r  Optional[IRNode]r   tuple[ReductionHint, _IntLike])
r  z<Callable[[Sequence[_IntLike], Sequence[_IntLike]], OpsValue]rI  r1  r3  r   rJ  r  r   z(Callable[[Sequence[_IntLike]], OpsValue])r   r+  r  r  rJ  r  r  r  r  r"  rI  r"  r3  r?   rK  rC   r  r.  r   rb   r3  r   r   r  r   #Union[_NumLike, Sequence[_NumLike]])r  r^   rs  r   rK  rC   r   rC   )r  r-  rI  r1  r  r^   r  r^   r  r^   r  r1  r   Callable[..., object])r  @Callable[[Sequence[sympy.Expr], Sequence[sympy.Expr]], OpsValue]r  r"  r  r"  r  Sequence[Integer]r  r4  r   r3  )r   r+  r  r  rJ  r  r  r  r  r"  r  r"  r  
list[Expr]r  list[Integer]r3  r?   r  r^   rK  rC   r   rb   )r   r+  r  r  rJ  r  r  r  r  r"  rI  r"  r3  r?   r  r^   rK  rC   r   rb   )r   r+  r  r  rJ  r  r  r  r  r"  r  r"  r  r6  r  r6  r3  r?   rK  rC   r   rb   ) rq   rr   rs   rt   ru   r  r  r  r  r  rY  r]  r  r  r  rJ  r  r  r  rC   rh  r  r  r  r  r  r  r%  r  r  rv   r  r  s   @rx   rH  rH  $  s   ((!!!!N H

%#S"S 3S 	S
 )S 
S=
K

  (,aaa a *	a
 #a -a >a a %a 
(a aF -N-,- - 	-
 
2- -^  )6(=(='+Q
Q
 Q
 	Q

 %Q
 Q
 )Q
 &Q
 &Q
 %Q
 
Q
 Q
f $/	, < DD$/D	,D D %(:G	   !'! -! "	!
 ! ! 5! 
! !F P ( $2	
 & 0 
J 8 =
=
 =
 	=

 '=
 (=
 $2=
 =
 ,=
 &=
 =
 &=
 
=
 =
~ $
$
 $
 	$

 %$
 $
 )$
 &$
 $
 &$
 
$
 $
L $
$
 $
 	$

 %$
 ($
 $2$
 "$
 ,$
 &$
 &$
 
$
 $
rw   rH  c                  z   ^  \ rS rSr% S\S'                     SU 4S jjr          SS jrSrU =r$ )	MultiOutputReductioni  r   output_indexc
                   >^ [        T5      (       a  T4m[        T5      S:X  a  TS   n
O      SU4S jjn
[        TU ]  UUU
UUUUUS9  Xl        g )Nr1   r   c                4   >^ ^ [        U U4S jT 5       5      $ )Nc              3  4   >#    U  H  o" TT5      v   M     g 7fr   ro   )r  r   r   reduction_idxs     rx   r  @MultiOutputReduction.__init__.<locals>.loader.<locals>.<genexpr>  s     HiR]33i   )r}   )r   r=  	inner_fnss   ``rx   r  -MultiOutputReduction.__init__.<locals>.loader  s     HiHHHrw   rg  )r   r"  r=  r"  r   ztuple[OpsValue, ...])callabler   r  __init__r9  )r;  r   r  r@  r  rI  r3  rJ  rK  r9  r  r  s      `       rx   rC  MultiOutputReduction.__init__  s     I"I y>Qq\FI#I4BI%I
 	-)) 	 		
 )rw   c           	     P   [         R                  " U R                  U R                  U R                  U R                  X45      5      n[        U[        [        45      (       d   [        U5       5       eXPR                     n[         R                  " U=(       d    SU" U5      U5      $ r  )rT   rX  r   rJ  r3  r  r{   r}   r|   r   r9  rY  )r;  r  r  r  rZ  r   r   s          rx   rY  $MultiOutputReduction.store_reduction  s     JJNNMM$/	
 &5$-00CT&\NC0(()"";#;)WT]ERRrw   )r9  )r   r+  r  r  r@  z)Union[INNER_FN_TY, Sequence[INNER_FN_TY]]r  r4  rI  r4  r3  r?   rJ  r  rK  rC   r9  r   r+  )	rq   rr   rs   rt   ru   rC  rY  rv   r  r  s   @rx   r8  r8    s    #)#) #) =	#)
 "#) ,#) &#) #) &#) #)JS"S 3S 	S
 )S 
S Srw   r8  c                  j    \ rS rSr\\R                  S4                   SS jj5       rSrg)OnlineSoftmaxReductioni  Nc
           	        ^^^^^^^ [        UUUUUUU4S j[        U5       5       5      n
U
 H  nUR                  5         M     U
$ )z.
Create the reduction disregarding splitting.
c              3  p   >#    U  H+  n[         R                  [        TTTTTS TTU5	      5      v   M-     g7f)r  N)rb   r  r8  )	r  
output_idxr   r  r  r  rK  rI  rJ  s	     rx   r  0OnlineSoftmaxReduction.create.<locals>.<genexpr>%  sO      
 0
 $$+"
  0s   36)r}   r   r  )r  r   r  rJ  r  r  rI  
num_outputrK  r  resultsr   s    `````` `   rx   r  OnlineSoftmaxReduction.create  sB       
 
 $J/
 
  AIIK rw   ro   )r   r+  r  r  rJ  r  r  r  r  r"  rI  r"  rM  r   rK  rC   r  r.  r   Sequence[TensorBox])	rq   rr   rs   rt   r  rC   rh  r  rv   ro   rw   rx   rH  rH    s     )6(=(='+!! ! 	!
 %! ! )! ! &! %! 
! !rw   rH  c                      \ rS rSr\\R                  4               SS jj5       r\      SS j5       r	\                  S	S j5       r
Srg)
WelfordReductioni:  c                  ^^^^^ US;   d   e[         R                  R                  R                  [	        T5      5      nS
UUU4S jjn	US:X  a  U	" S5      n
U	" S5      nU	" S5      nXU4$ US:X  aD      SUUUU4S jjmUS:X  a  T" US   5      U	" S5      U	" S5      4$ [        U4S jU 5       5      $ [        R                  TTTUS   TTUUS9u  pU[        R                  :X  a  UnUS:  a  U R                  TTUTTUUU5      $ [        S	5       Vs/ s H)  n[        R                  [        TTUTTUTUU5	      5      PM+     nnU H  nUR                  5         M     U$ s  snf )N)r  r6  c                V   >^  SUU 4S jjn[         R                  TTU[        T5      S9$ )Nc                2   > [         R                  " TT5      $ r   r  )r   r   r  s    rx   r  8WelfordReduction.create.<locals>.const.<locals>.inner_fnK  s    || rw   r  r   r"  r   rU   r  r  r|   )r  r  r   r   r  s   ` rx   const&WelfordReduction.create.<locals>.constJ  s7      ##!F|	 $  rw   r   r1   c                V   >^  SU U4S jjn[         R                  TTU[        T5      S9$ )Nc                r   > T Vs/ s H  n[         R                  R                  PM     nnT" X5      $ s  snf r   r  )r   r   r  r  rI  s      rx   r  7WelfordReduction.create.<locals>.copy.<locals>.inner_fnc  s2    =M&N=Muww||=MO&N!#77 'Or  r  rW  rX  )r  r  r   r   r  rI  s   ` rx   copy%WelfordReduction.create.<locals>.copy`  s7    8 8 !''!%<	 (  rw   r  c              3  4   >#    U  H  nT" U5      v   M     g 7fr   ro   )r  r   r^  s     rx   r  *WelfordReduction.create.<locals>.<genexpr>q  s     :	"T"XX	r?  )r3  r  r   )r  r   r   rb   )r  z4Callable[[Sequence[Expr], Sequence[Expr]], OpsValue]r   rb   )rV   r   r   r  rR   r}   rH  r  rC   rh  r  r   rb   r  rR  r  )r  r   r   r@  r  rI  r3  rK  r  rY  meanm2weightr  r  rK  rN  r   r^  s    `` ``            @rx   r  WelfordReduction.create;  s    !FFFF''**33MBR4ST	 	 a8DqB1XFV##aL  !11IaL)58U1X==:	:::&  **aL)+ + 	
 ]222!N19(( 	 	2 $Ah
 '
  $""
 ' 	 
  AIIK %
s   0E#c                    g)N)r   r   r   ro   r  s     rx   r  WelfordReduction.default_value  s     rw   c	                  ^ ^^^^^^ [        T5      m[        R                  R                  R	                  [
        R                  " TT-  S5      5      (       + n	U	(       aB  US:w  a<          S
U4S jjn
T R                  UTUS   [        U
SS9[        U
SS94UTSTUS9$ [        TTS-
  -   T5      m[        R                  UT[        UU UUU4S jU 5       5      / UQTPT/UU5      nU H  nUR                  5         M             SS jm[        R                  R                  R                  [        U5      5      nT R                  TX5      n[        R                  UT[        U4S	 jU 5       5      UT/SU5      $ )r  r   r6  c                2   > [         R                  " UT5      $ r   r  )r   r=  r   r   s      rx   r  4WelfordReduction.create_multilayer.<locals>.constant  s     ||E511rw   r   r1   )r   r   r@  r  rI  r3  r  rK  c           
   3  L   >#    U  H  nTR                  UTTTTS S9v   M     g7f)r   )r  N)r  )r  r  r  r  r  rI  r  s     rx   r  5WelfordReduction.create_multilayer.<locals>.<genexpr>  s>      
 (F ++$# ,  (s   !$c                    U" / U QUQ5      $ r   ro   )r   r  r  s      rx   intermediate_loader_fnBWelfordReduction.create_multilayer.<locals>.intermediate_loader_fn  s    
 4E4O455rw   c              3  T   >#    U  H  n[        TUR                  5       S 9v   M     g7f))r  N)r
   r  )r  r   rn  s     rx   r  rl    s&      &A .q}}G&r  )r   r"  r=  r"  r   r   r   rU   )r   r"  r  r"  r  r-  r   rU   )rR   rV   r   r   r  r   r  r  r
   r.   rR  r  r}   r  r   r  )r  r   r   r@  r  rI  r3  r  rK  r  r  intermediatesr   rs  r  rn  r  s   ` `  ` `      @@@rx   r  "WelfordReduction.create_multilayer  s     ((89((@@HH_u,a0
 
	 +<<2#24B2KN22
 ((aLHA.HA.
 !10- )   o;UC
(// 
 
 (
 
 feL#
& AIIK 	6!	6+	6 9	6 		6 WW%%//f0EF
99:
  && &  G
 	
rw   ro   N)r   r+  r   r  r@  Sequence[Callable[..., Any]]r  r6  rI  r6  r3  r?   rK  rC   r   rP  r0  )r   r+  r   r  r@  rs  r  r6  rI  r6  r3  r?   r  r^   rK  rC   r   rP  )rq   rr   rs   rt   r  rC   rh  r  rJ  r  r  rv   ro   rw   rx   rR  rR  :  s    )6(=(=vv v 0	v
 v (v &v &v 
v vp $/	, 
 Z
Z
 Z
 0	Z

 Z
 (Z
 &Z
 Z
 &Z
 
Z
 Z
rw   rR  c                    ^  \ rS rSr% S\S'   S\S'   S\S'   S\S'   S	\S
'   S\S'   S\S'   S\S'   S U 4S jjrS!U 4S jjr          S"S jrS#S jrS$S jr	S%S jr
S%S jrS&S jrS'S jrS S jr\\R"                  4SS.                   S(S jjj5       r\                  S)S j5       rSrU =r$ )*Scani  r6  scan_rangesr   =Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]r  zFCallable[[Sequence[_IntLike], Sequence[_IntLike]], Sequence[_IntLike]]r   rC   rK  r   r9  tuple[torch.dtype, ...]dtypestuple[Callable[..., Any], ...]r@  c                   > [         TU ]  5       [        5       R                  " S U R                   5       6 -  [        5       R                  " S U R
                   5       6 -  $ )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  s     rx   r  0Scan.get_unbacked_symbol_uses.<locals>.<genexpr>,       "VEU#8#;#;EUrL  c              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  s     rx   r  r}  -       "OY#8#;#;YrL  )r  r  r,   r  rv  r   r  s    rx   r  Scan.get_unbacked_symbol_uses&  sW    
 G,.l  "VTEUEU"VWXl  "OTYY"OPQ	
rw   c                   > [        U R                  5      [        U R                  5      -   [        U R                  5      :X  d   e[        TU ]  5         g r   )r   r  rv  r   r  rC  r  s    rx   rC  Scan.__post_init__0  =    4;;#d&6&6"773tyy>IIIrw   c                "  ^ U R                  X45      m[        U4S jU R                   5       5      n[        R                  " U R
                  U R                  U5      n[        R                  " U=(       d    SU" T5      X`R                     5      $ )Nc              3  2   >#    U  H  o" T5      v   M     g 7fr   ro   r  r  r   s     rx   r  'Scan.store_reduction.<locals>.<genexpr><       D^x}}^   r  )	r   r}   r@  rT   rn  ry  r  r  r9  )r;  r  r  r  	scan_varsr   resultr   s          @rx   rY  Scan.store_reduction4  sk     ll4+DT^^DD$++t?yy$9gclF;L;L4M
 	
rw   c                    g)Ncustomro   rB  s    rx   r  Scan.get_reduction_typeB  s    rw   c                    U R                   $ r   )rv  rB  s    rx   r  Scan.get_reduction_sizeF  rY  rw   c                    U R                   $ r   r   rB  s    rx   r   Scan.get_sizeI      yyrw   c                    U R                   $ r   r  rB  s    rx   r  Scan.get_pointwise_sizeL  r  rw   c                X    [        U R                  5      [        U R                  5      -   $ r   )r   r  rv  rB  s    rx   r]  Scan.index_lengthO  !    4;;#d&6&6"777rw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      nU R                  X5      nU4$ r   )r  r  rv  r0   r`  r   r;  r   rb  r   s       rx   r  Scan.inner_fn_argsR  C    DKK(T--t}}=ll5)vrw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      nU R                  X5      n[        U R                  U5      $ r   )r  r  rv  r0   r`  r   r8   r  r  s       rx   r  #Scan.inner_fn_free_unbacked_symbolsX  M    DKK(T--t}}=ll5),T]]C@@rw   T)can_fallback_to_atenc                 ^^^ / US T QUTS-   S  QmUT   /m[         R                  R                  U[        R                  5      (       d  S /[        U5      -  $ [        U5      S:  aB  [         R                  R                  U[        R                  5      (       d  S /[        U5      -  $ [         R                  R                  n
U
R                  [        T5      5      n[        U5      [        U5      :X  d   eU
R                  [        R                  " US5      5      (       a=  [        [        U5      5       Vs/ s H  n[        R                  UX,   X<   US9PM     sn$ U R!                  UUS   US   TTTUUS9u  p}["        nUS:  at  [$        R&                  R(                  S L =(       d    [*        =(       a	    [,        S:  =(       a    [        U5      S:H  nU(       d  U(       a  S /[        U5      -  $ SnO[.        nSUUU4S jjn[        [        U5      5       Vs/ s H/  n[0        R                  U" S	UX,   UX<   UUTTUUUUS.U	D65      PM1     nnU H  nUR3                  5         M     U$ s  snf s  snf )
Nr1   r  r   )r   r   r  axispointwise_rangesrv  r  
scan_numelz3.3.0c                   > [        U5      [        T5      :X  d   e[        U 5      [        T5      :X  d   e/ U S T QUQU TS  Q$ r   r   )r   
scan_indexr  r  rv  s     rx   r   Scan.create.<locals>.reindex  S    z?c+&6666u:%5!6666>U5D\>J>tu>>rw   )r   r   ry  r  r@  r   r  rv  r  r   rK  r9  )r   r"  r  r"  r   r5  ro   )rV   r   r  r4   SCANr   TUPLE_REDUCTIONr   r  rR   r  r   Ler   r  r  r  ru  r   versionhip
has_tritontriton_version	SplitScanrb   r  )r  r   ry  r@  r   r  r  rK  r  r   r   r  r9  r  	scan_typesupports_splitr   rN  r  r  rv  s        `             @@rx   r  Scan.create^  s    =T%4[<4q
+;<Dzlww""6>+>+>??6CK''v;?177#6#6N22$
 $
 6CK''77##&&}['AB
6{c)n,,, ++EHHZ,CDD %*#f+$6 %7L   ! .&4	 !  %7  &)^^)q\-#!! &4 	&
" 	>!!T)Wj.V^w=V%v;!#  "' 6CK//!"J%		? 	?. !&c&k 2%
$ !3#  ! .!&4'+ +)##1!- " !3% 	 
* FNN  N
s   ;$I3 6I8c	                L   ^^ SUU4S jjn	[         R                  UUUU	UUSUS9$ )Nc                ,   > T" / U S T QUQU TS  Q5      $ r   ro   )r   r=  r  r  s     rx   r  #Scan.num_splits.<locals>.wrapper_fn  s*    Fc%4jF=F3tu:FGGrw   rn  )r   r  rJ  r  r  rI  r3  r  )r   r"  r=  r"  r   rU   )rH  r  )
r  r   r   r  r  r  rv  r  r  r  s
      ``     rx   r  Scan.num_splits  sA    	H 	H ###(!& $ 	
 		
rw   ro   rB  r  )
r  r&  r  z%Callable[[Sequence[_IntLike]], Never]r  r"  r  r	  r   r   r%  rD  r!  r@  r  )r   r+  ry  rx  r@  z+tuple[Callable[[Sequence[Expr]], Any], ...]r   r6  r  r   r  rw  rK  rC   r  rm   r   r   r   Sequence[Optional[TensorBox]])r   r+  r   r  r  r-  r  r   r  r6  rv  r6  r  rw  r  r   r   r/  )rq   rr   rs   rt   ru   r  rC  rY  r  r  r   r  r]  r  r  r  rC   rh  r  r  rv   r  r  s   @rx   ru  ru    s   
MMSS!!##--
 
"
 7
 	

 $
 

 8A  )6(=(=_ &*__ (_ ?	_
 _ _ R_ &_ #_ _ 
'_ _B 

 
 7	

 
 (
 #
 R
 
 
(
 
rw   ru  c                      \ rS rSrSrg)r  i  ro   Nrq   rr   rs   rt   rv   ro   rw   rx   r  r    s    rw   r  c                  P  ^  \ rS rSr% S\S'   S\S'   S\S'   S\S'   S	\S
'   S\S'   S\S'   S\S'   S\S'   SU 4S jjrSU 4S jjr          S S jrS!S jrS"S jr	S"S jr
S"S jrS#S jrS$S jrSS jr\\R"                  4                   S%S jj5       rSrU =r$ )&Sorti  r6  sort_rangesr   z:Callable[[Sequence[Expr], Sequence[Expr]], Sequence[Expr]]r   rC   rK  r   r9  rx  ry  rz  r@  rm   stable
descendingc                   > [         TU ]  5       [        5       R                  " S U R                   5       6 -  [        5       R                  " S U R
                   5       6 -  $ )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  s     rx   r  0Sort.get_unbacked_symbol_uses.<locals>.<genexpr>  r~  rL  c              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  s     rx   r  r    r  rL  )r  r  r,   r  r  r   r  s    rx   r  Sort.get_unbacked_symbol_uses  sU    G,.l  "VTEUEU"VWXl  "OTYY"OPQ	
rw   c                   > [        U R                  5      [        U R                  5      -   [        U R                  5      :X  d   e[        TU ]  5         g r   )r   r  r  r   r  rC  r  s    rx   rC  Sort.__post_init__  r  rw   c                6  ^ U R                  X45      m[        U4S jU R                   5       5      n[        R                  " U R
                  XPR                  U R                  5      n[        R                  " U=(       d    SU" T5      X`R                     5      $ )Nc              3  2   >#    U  H  o" T5      v   M     g 7fr   ro   r  s     rx   r  'Sort.store_reduction.<locals>.<genexpr>	  r  r  r  )
r   r}   r@  rT   sortry  r  r  r  r9  )r;  r  r  r  rZ  r   r  r   s          @rx   rY  Sort.store_reduction  so     ll40DT^^DD$++v{{DOOLyy$9gclF;L;L4M
 	
rw   c                    g)Nr  ro   rB  s    rx   r  Sort.get_reduction_type	  s    rw   c                    U R                   $ r   )r  rB  s    rx   r  Sort.get_reduction_size	  rY  rw   c                    U R                   $ r   r  rB  s    rx   r   Sort.get_size	  r  rw   c                    U R                   $ r   r  rB  s    rx   r  Sort.get_pointwise_size	  r  rw   c                X    [        U R                  5      [        U R                  5      -   $ r   )r   r  r  rB  s    rx   r]  Sort.index_length	  r  rw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      nU R                  X5      nU4$ r   )r  r  r  r0   r`  r   r  s       rx   r  Sort.inner_fn_args	  r  rw   c                    U R                  U R                  5      nU R                  U R                  [        R                  5      nU R                  X5      n[        U R                  U5      $ r   )r  r  r  r0   r`  r   r8   r  r  s       rx   r  #Sort.inner_fn_free_unbacked_symbols!	  r  rw   c	                  ^^^ / US T QUTS-   S  QmUT   /m[         R                  R                  U[        R                  5      (       d  S /[        U5      -  $ [         R                  R                  n
U
R                  [        T5      5      nSn[        R                  R                  =(       a%    U
R                  [        R                  " X5      5      nU(       d  S /[        U5      -  $ [        U5      [        U5      :X  d   eU
R                  [        R                  " US5      5      (       a=  [        [        U5      5       Vs/ s H  n[         R#                  UX.   X>   US9PM     sn$ SUUU4S jjn[        [        U5      5       Vs/ s H3  n[$        R#                  ['        SUX.   UX>   UUTTUUUUUS.U	D65      PM5     nnU H  nUR)                  5         M     U$ s  snf s  snf )Nr1   r  r  c                   > [        U5      [        T5      :X  d   e[        U 5      [        T5      :X  d   e/ U S T QUQU TS  Q$ r   r  )r   
sort_indexr  r  r  s     rx   r   Sort.create.<locals>.reindexV	  r  rw   )r   r   ry  r  r@  r   r  r  r   rK  r9  r  r  )r   r"  r  r"  r   r5  ro   )rV   r   r  r4   SORTr   r   r  rR   r2   r   persistent_reductionsr  r   r  r   r  r  rb   r  r  )r  r   ry  r@  r   r  r  r  rK  r   r   
sort_numel
max_rblockis_persistent_kernelr9  r   rN  r  r  r  s        `            @@rx   r  Sort.create'	  s    =T%4[<4q
+;<Dzlww""6>+>+>??6CK''77##&&}['AB
 
MM// S00*1QR 	 $6CK''6{c)n,,, ++EHHZ,CDD %*#f+$6 %7L   ! .&4	 !  %7 	? 	?0 !&c&k 2'
& !3%  ! .!&4'+ +##1!-!) $ !3' 	 
, FNN  Q
s   $G&:G+ro   rB  r  )
r  r&  r  r/  r  r"  rZ  r"  r   r   r%  r!  r@  r,  )r   r+  ry  rx  r@  z'tuple[Callable[[list[Expr]], Any], ...]r   r6  r  r   r  rm   r  rm   rK  rC   r   r   r   r  )rq   rr   rs   rt   ru   r  rC  rY  r  r  r   r  r]  r  r  r  rC   rh  r  rv   r  r  s   @rx   r  r    s*    
GG!!##--L
 
"
 2
 	

 '
 

 8A  )6(=(=LL (L ;	L
 L L L L &L L 
'L Lrw   r  c                :     [        U SS9  g! [         a     gf = f)NFfreezeT)r!  rt  r   s    rx   r   r   w	  s&    a. s   
 
c                     [        U SS9u  pUR                  5       (       a  UR                  5         UR                  5       $ ! [         a     gf = fNFr  )r!  should_pad_stridespad_stridesis_contiguousrt  )r   _bufferr   s      rx    is_contiguous_storage_and_layoutr  	  sS    /%@ $$&& ##%% s   A A 
AAc           	        [        U [        5      (       a  [        U R                  UUUUUS9$ [        U [        5      (       a5  [        U R                  UUUUUS9u  pgX R                  R                  5       4$ [        U [        5      (       a  U(       as  U(       a6  U R                  5         U R                  5       R                  5       (       d   eO6Ub  U R                  X4S9  O#Ub  U R                  XTS9  OU R                  5         [	        U 5      U R                  5       4$ [        U [        5      (       a#  [        U R                  US9u  pXR                  4$ [        e)z
Try to simplify x into a StorageBox and a Layout.

allow_padding only affect how we apply stride_order. When allow_padding
is True, we have the freedom to add padding when applying the stride_order.
r  want_contiguousstride_orderr  r  r  r  )r{   rb   r!  r  
StorageBoxr   Bufferr  r  r  r  r  r$  r   rt  )	r   r  r  r  r  r  r   r   buffers	            rx   r!  r!  	  sJ    !Y$FF+%''
 	
 !Z  )FF+%''
	 &&##%%%!V!||~335555)11  2  *22! 3  !!}alln,,!_%% *FF
	 xx
rw   c                ^     [        U SS9u  p#UR                  U5      $ ! [         a     gf = fr  )r!  is_stride_orderedrt  )r   r  r  r   s       rx   "is_stride_order_storage_and_layoutr  	  s8    /%@''55 s    
,,c                      \ rS rSr% S\S'   SS jrSS jrSS jrSS jr\	SS j5       r
S S	 jrS!S
 jrS"S jrS#S jrS$S jrS%S jrS&S jrS'S jrS rS rS&S jrS&S jrS(S jrS)S jrS rS*S jrSrg)+BaseViewi	  rc   r  c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  !BaseView.get_unbacked_symbol_uses	      yy1133rw   c                    [        SU  35      e)Nzmake_reindexer NYI on rU  rB  s    rx   make_reindexerBaseView.make_reindexer	  s    !$:4&"ABBrw   c                p   ^^ U R                   R                  5       mU R                  5       mSUU4S jjnU$ )Nc                    > T" T" U 5      5      $ r   ro   r   innerr   s    rx   r  &BaseView.make_indexer.<locals>.indexer	      &&rw   )r   r"  r   r   )r  r  r  )r;  r  r	  r   s     @@rx   r  BaseView.make_indexer	  s4    		&&(%%'	' 	' rw   c                p   ^^ U R                   R                  5       mU R                  5       mSUU4S jjnU$ )Nc                    > T" T" U 5      5      $ r   ro   r  s    rx   r  $BaseView.make_loader.<locals>.loader	  r  rw   rW  )r  r  r  )r;  r  r	  r   s     @@rx   r  BaseView.make_loader	  s4    		%%'%%'	' 	' rw   c                6    U R                   R                  5       $ r   )r  r   rB  s    rx   r   BaseView.dtype	  s    yy""$$rw   c                6    U R                   R                  5       $ r   r  r   rB  s    rx   r   BaseView.get_layout	      yy##%%rw   c                6    U R                   R                  5       $ r   r  r   rB  s    rx   r   BaseView.get_device	  r  rw   c                    g r   ro   rB  s    rx   rW  BaseView.get_origin_node	  r]  rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.get_name	      yy!!##rw   c                "    U R                  5       $ r   r  rB  s    rx   r  BaseView.get_pointwise_size	      }}rw   c                8    U R                   R                  U5      $ r   r  r  r  s     rx   r  BaseView.mark_reuse 
      yy##E**rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.has_exceeded_max_reads
      yy//11rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.realize
      yy  ""rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.realize_hint	
      yy%%''rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.get_storage_numel
      yy**,,rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  BaseView.is_extern
      yy""$$rw   c                6    U R                   R                  5       $ r   )r  is_module_bufferrB  s    rx   r<  BaseView.is_module_buffer
      yy))++rw   c                6    U R                   R                  5       $ r   r  rO  rB  s    rx   rO  BaseView.get_read_names
      yy''))rw   c                    [         R                  " [        SS5         [        U R	                  5       U R                  5       5      R                  sS S S 5        $ ! , (       d  f       g = fr  )r   r   r   r:   r  r   r  rB  s    rx   rN  BaseView.get_reads
  sD    \\.*:DA&  " e	 BAAs   2A
A'c                z    U n[        U[        5      (       a#  UR                  n[        U[        5      (       a  M#  U$ r   )r{   r  r  )r;  r   s     rx   r  BaseView.unwrap_view
  s1    H%%A H%%rw   c                    U R                  5       n[        R                  " [        SU5      " U5      n[	        UU R                  5       UU R                  5       S9$ r  )r  r   r   r  r  r   r   r  s      rx   r  BaseView.constant_to_device%
  sN    !!#n.?HP.."==?	
 	
rw   ro   NrB  )r   z*Callable[[Sequence[Expr]], Sequence[Expr]]r.  r,  r  r  r(  r  r2  r!  r5  r   r%  r  r?  rF  )rq   rr   rs   rt   ru   r  r  r  r  rM  r   r   r   rW  r  r  r  r  r  r  r  r  r<  rO  rN  r  r  rv   ro   rw   rx   r  r  	  s    
L4C % %&&$+2#(-%,*	
rw   r  c                  P    \ rS rSr% S\S'   \S 5       r\S 5       rS
S jr	S r
Srg	)r   i1
  r5  r   c                t   [         R                  R                  n[        [	        [
        R                  U5      5      nU R                  5       nS/[        U5      [        U5      -
  -  [        U5      -   n[        U5      [        U5      :X  d   e[        [        U5      5       H  nX   S:X  a  X4   c   eX4   X'   M  X4   bN  [         R                  R                  R                  R                  [
        R                  " X4   S5      SS9(       a  Mo  UR                  X   X4   -
  SS9S:X  a  M   S5       e   U$ )	zReplace `-1` with correct sizesNr  r1   Tsize_obliviousr   fallbackzKBroadcast failed in ExpandView({x.get_size()}, {new_size}) on dimension {i})rV   r   r   r|   rg  r   r  r   r   r   r   evaluate_exprr  r   )r   new_sizer   old_sizer   s        rx   _normalize_sizeExpandView._normalize_size5
  s    77##ELL(34::<6S]S]:;d8nL8}H---s8}%A{b {...&k$(8(8(B(B(P(Pa( )Q )   ))(+*Ca)PTUU aU &" rw   c           	        U R                  X5      n[        U5      (       Ga3  [        U5      u  p4[        U5      [        UR                  5      -
  nUS:  d   e[
        R                  R                  /U-  n[        UR                  UR                  5       H|  u  pxUR                  [        R                  R                  R                  R                  [
        R                   " US5      SS9(       d  UO[
        R                  R                  5        M~     [#        UR$                  UR&                  [)        U5      UUR*                  5      n	[-        X9S9$ [/        XS9$ )Nr   r1   TrK  r  )r  r   )rR  r   r!  r   r   r   r  r  r   r   r  rV   r   r   r   rO  r  r"  r   r   r|   r#  r$  r   )
r  r   rP  r'  r(  skipr)  r   r   r*  s
             rx   r  ExpandView.createP
  s   &&q3 ##"7":Gx=3z#77D199'',,$.J #J$5$5z G!!77++55CCq)$ D    !H %!!  X!!J #CCq00rw   c                    U R                   $ r   r  rB  s    rx   r   ExpandView.get_sizel
  r  rw   c                   ^^ U R                  5       nU R                  R                  5       m[        U5      [        T5      -
  mUU4S jnU$ )Nc                   > [        U TS  5      n [        U 5      [        T5      :X  d   e[        [        T5      5       H*  nTU   S:X  d  M  [        R                  R
                  X'   M,     U $ r  )r|   r   r   r   r  r  )r   r   actualrU  s     rx   r   *ExpandView.make_reindexer.<locals>.reindext
  s]    tu&Eu:V,,,3v;'!9>$ww||EH ( Lrw   )r   r  r   )r;  targetr   r[  rU  s      @@rx   r  ExpandView.make_reindexero
  s>    ##%6{S[(	 rw   ro   Nr!  )rq   rr   rs   rt   ru   rJ  rR  r  r  r   r  rv   ro   rw   rx   r   r   1
  s8    
 4 1 16rw   r   c                  P    \ rS rSr% S\S'   \S 5       r\S 5       rS
S jrS r	Sr
g	)PermuteViewi
  r5  dimsc           
        U R                  U5      n[        U5      [        [        [        U5      5      5      :X  d   e[	        U5      (       a}  [        U5      u  p4[        UR                  UR                  U Vs/ s H  oTR                  U   PM     snU Vs/ s H  oTR                  U   PM     snUR                  5      n[        X6S9$ [        XS9$ s  snf s  snf )Nr  )r  ra  )_map_neg_dimsr,   r   r   r   r!  r"  r   r   r   r   r#  r$  r`  )r  r   ra  r'  r(  r   r*  s          rx   r  PermuteView.create
  s      &$:eCI.>#???? ##"7":G$!!  -12T#T2/34t!""1%t4!!J #CC-- 34s   :CCc                X    U Vs/ s H  o"S:  a  UO[        U5      U-   PM     sn$ s  snf r  r  )r  ra  r  s      rx   rc  PermuteView._map_neg_dims
  s+    @DEaxSY_4EEEs   'c                   [        U R                  U R                  5      5      [        [        [	        U R                  5      5      5      :X  d   eU R
                  R                  5       nU R                   Vs/ s H  o!U   PM	     sn$ s  snf r   )r,   rc  ra  r   r   r  r   )r;  r   r   s      rx   r   PermuteView.get_size
  sq    $,,TYY78J#dii.!=
 
 	
 
 yy!!#!%+AQ+++s   8B	c                N  ^ [        U R                  5       VVs0 s H  u  pX!_M	     snnm[        [        U R                  5      5       Vs/ s H  nTU   PM
     snm[	        T5      [	        [        [        U R                  5      5      5      :X  d   eU4S jnU$ s  snnf s  snf )Nc                8   > T Vs/ s H  oU   PM	     sn$ s  snf r   ro   )r   r   invs     rx   r   +PermuteView.make_reindexer.<locals>.reindex
  s    &)*c!Hc***s   )r   ra  r   r   r,   )r;  r   r  r   rk  s       @rx   r  PermuteView.make_reindexer
  s     )$)) 45 4qt 45$S^454!s1v45#*U3tyy>-B"CCCC	+  65s   BB"ro   Nr!  )rq   rr   rs   rt   ru   r  r  rc  r   r  rv   ro   rw   rx   r`  r`  
  s:    
. ." F F,rw   r`  c                  J    \ rS rSr\SS.S j5       r\SS j5       rS	S jrSr	g)
SqueezeViewi
  N)r  c          	        [        U5      (       Ga!  [        U5      u  p4/ n/ nUb=  [        U[        5      (       d   S5       eSU::  a  U[	        UR
                  5      :  d   e[        [        UR
                  UR                  5      5       Hm  u  nu  pUc,  US:w  a$  UR                  U5        UR                  U	5        M5  M7  Xr:w  a$  UR                  U5        UR                  U	5        M`  US:X  a  Mh   S5       e   [        UR                  UR                  UUUR                  5      n
[        X:S9$ Uc;  [        R!                  XR#                  5        Vs/ s H  oS:w  d  M
  UPM     sn5      $ UR#                  5       U   S:X  d   e[        R!                  U[        UR#                  5       5       VVs/ s H  u  p{Xr:w  d  M  UPM     snn5      $ s  snf s  snnf )Nzexpected integer dim argumentr   r1   zexpected squeezed size to be 1r  )r   r!  r{   r   r   r   r   r   r   r  r"  r   r   r#  r$  r
  r  r   )r  r   r  r'  r(  rP  r)  r   r   r   r*  r   s               rx   r  SqueezeView.create
  s    ##"7":GHJ!#s++L-LL+CxC#joo*>$>>>%.s:??JDUDU/V%W!>D;qy -"))&1 ! x -"))&1#qyJ*JJy &X %!!  !!J #CC;;;qjjl"El1f1l"EFF::<$)));;q1::<1H"U1HAH11H"UVV #F #Vs   	G
$G
6GGc                   ^^ U  Vs/ s H  oS:w  d  M
  UPM     nn[        U 5       VVs/ s H  u  p1US:w  d  M  UPM     snnm[        U 5      mSUU4S jjnX$4$ s  snf s  snnf )Nr1   c                   > [        U 5      [        T5      :X  d   U  ST 35       e[        R                  R                  /T-  n[	        TU 5       H	  u  p#X1U'   M     [        U5      $ )N )r   r   r  r  r   r}   )r   r  r   r   lengthnot_ones       rx   r   %SqueezeView.squeezer.<locals>.reindex
  sb    u:W-C%'/CC-/Igu-!"# .##rw   )r   zlist[sympy.Expr]r   ztuple[sympy.Expr, ...])r   r   )r   r   rP  r   r   ru  rv  s        @@rx   squeezerSqueezeView.squeezer
  sa    #.t!AvAt.!*4;AF1;T	$ 	$    /;s   	AAA A c                    [        S5      e)Nzuse SqueezeView.create())AssertionError)r;  r  s     rx   rC  SqueezeView.__init__
  s    788rw   ro   )r   rE  r  )
rq   rr   rs   rt   r  r  rJ  rx  rC  rv   ro   rw   rx   ro  ro  
  s3    " #W #WJ ! !9rw   ro  c                  b    \ rS rSr% S\S'   S\S'   S rSS jrSS jr\r\	S	 5       r
SS
 jrSrg)GenericViewi
  r5  r   r  r   c                    U R                   $ r   )r   rB  s    rx   r  GenericView.make_reindexer
      ||rw   c                   [        [        U R                  5      5       Vs/ s H  n[        [        R
                  U5      PM     nn[        U R                  U5      5      nSSR                  [        [        U5      5       SU 3$ s  snf )Nzlambda , r   )r   r   r   rQ   r0   r  r|   r   ri  rg  r   )r;  r  	index_old	index_news       rx   reindex_strGenericView.reindex_str
  sv    CHTYYCX
CXa*4::q9CX 	 
 i01	3sI#6789+FF	
s   $Bc                z    U R                  U R                  SU R                   3SU R                  5        3/5      $ )Nsize=zreindex=)rm  r  r   r  rB  s    rx   r  GenericView.__str__
  s=    YY%		{+x8H8H8J7K-LM
 	
rw   c                $    U " U[        U5      US9$ )Nr  r   r   )r|   )r  r   rP  r   s       rx   r  GenericView.create
  s    X@@rw   c                    U R                   $ r   r  rB  s    rx   r   GenericView.get_size  r  rw   ro   Nr2  r!  )rq   rr   rs   rt   ru   r  r  r  r  r  r  r   rv   ro   rw   rx   r~  r~  
  s>    
G

 HA Arw   r~  c                  d    \ rS rSr\S 5       r\S 5       r\S 5       r\S 5       r	\S 5       r
Srg)	r
  i  c                   [         R                  " U 5      n [         R                  " U5      n[        R                  R                  R
                  R                  nU" [         R                  " U S5      5      (       a  X-   n U $ r  )r   r  rV   r   r   r   rO  Lt)r   r   rO  s      rx   handle_negative_indexView.handle_negative_index  s[    ll3||D!((22@@#q)***C
rw   c                B  ^	 [        U[        [        45      (       d   eU R                  UR	                  5       U5      u  m	n[
        R                  R                  R                  T	U5      (       a  U$ Sn[        [        T	5      5      S:  d  [        [        U5      5      S:  a  SnSU;   a  U	4S jnU " U[        U5      US9$ [        U5      (       d  U(       a  U(       aG  [        U5      (       d7  [        R                  U[        R                  UR	                  5       5      5      n[!        USS9u  pV[#        UR$                  UR&                  U[        R                  U5      UR(                  5      n[+        XWS9$ U R-                  T	U5      nU " U[        U5      US9$ )NFr   Tc                4   > [        S/[        T5      -  5      $ r  )r}   r   )r   rQ  s    rx   fake_reindex!View.create.<locals>.fake_reindex"  s    aS3x=011rw   r  )r  r  )r{   r}   r|   resolve_negative_sizer   rV   r   r   statically_known_list_equalsr   r&   r  ExternKernelrequire_exact_stridesr   r   r!  r"  r   r   r#  r$  r  )
r  r   rP  unbacked_symbols_in_sizesr  r'  r(  r*  r   rQ  s
            @rx   r  View.create  sl   (UDM2222 66qzz|XN( 77888LLH$)!%h/014(23a7(,%=2 ADNLII-a004M(2RST2U2U !66~88F #84"PG$!!  11(;!!J #CC--hAX@@rw   c                P   U Vs/ s H,  n[         R                  R                  R                  U5      PM.     nnU  Vs/ s H,  n[         R                  R                  R                  U5      PM.     n n[	        U5      n[        [        U5      5       HI  nX   S:X  d  M  [        R                  R                  X'   [        [        U 5      [        U5      5      X'     O   [         R                  R                  R                  [        U 5      [        U5      5        X4$ s  snf s  snf )Nr  )rV   r   r   r  r|   r   r   r   r  Oner-   rR   guard_equals)rQ  rP  r   r   s       rx   r  View.resolve_negative_size?  s    :BC(QAGG$$--a0(C:BC(QAGG$$--a0(C>s8}%A{b #ggkk&}X'>h@WX	 & 	
%%mH&=}X?VW!! DCs
   3D3D#c                     U R                  X5      nU$ ! [        [        4 a=    [        U5      /nU R                  X5      nU R                  XB5      n[	        XV5      n U$ f = fr   )_dynamic_reshape_indexerr{  
IndexErrorrR   r   )r  rQ  rP  r   flatr   r   s          rx   r  View.dynamic_reshape_indexerN  sp    	:228FG  
+ 	:!(+,D33HCH33DCH%h9G	:s    A	A"!A"c                  ^^ [         R                  R                  R                  n[	        [        U5      5       Vs/ s H  n[        [        R                  U5      PM     snm[        [        TU5      5      n[        U 5      n/ mU(       GaK  U(       GaC  UR                  5       nUR                  5       u  pxUS:X  a=  TR                  [        R                  R                  5        UR                  Xx45        GOUS:X  a  UR                  U5        GOU" U5      U" U5      :X  a<  TR                  U5        [         R                  R                  R!                  X5        GOfU" U5      U" U5      :  a~  U" U5      U" U5      :  a1  UR                  5       u  pX-  U-   nX-  nU" U5      U" U5      :  a  M1  TR                  U5        [         R                  R                  R!                  X5        OU" U5      U" U5      :  a  [        R                  R"                  nUnTR                  [%        X{U5      5        X-  nU" U5      U" U5      :  aG  UR                  5       nTR                  [%        X{U5      5        X-  nXl-  nU" U5      U" U5      :  a  MG  [         R                  R                  R!                  X5        O[&        eU(       a
  U(       a  GMC  U(       al  UR                  5       n[         R                  R                  R!                  US5        TR                  [        R                  R                  5        U(       a  Ml  U(       aE  UR                  5       u  px[         R                  R                  R!                  US5        U(       a  ME  TR)                  5         [        T5      [        U 5      :X  d   eUU4S jnU$ s  snf )z7
Perform a reshape entirely by modifying indexing math
r1   c                   >^ [        U 5      [        T5      :X  d   [        U 5      [        T5      45       e[        [        TU 5      5      m[        U4S jT 5       5      $ )Nc              3  <   >#    U  H  n[        UT5      v   M     g 7fr   )rS   )r  r   replacementss     rx   r  AView._dynamic_reshape_indexer.<locals>.reindex.<locals>.<genexpr>  s     HiA|44is   )r   r~   r   r}   )r   r  r  	view_exprs    @rx   r   .View._dynamic_reshape_indexer.<locals>.reindex  sM    u:T*CSZT,CC*D% 01LHiHHHrw   )rV   r   r   r   r   r   rQ   r0   VIEWr|   r   r  r  r   r  r  r  r  r/   r{  reverse)rQ  rP  r   r   	stack_new	stack_oldsize_oldvarsize_newvar2	size_new2divisormodulusr   r  r  s                 @@rx   r  View._dynamic_reshape_indexerZ  s   
 GG$$..	 CHHBV
BVQ*499a8BV
 T8,-	N		I }}H%MMOMC1}  .  #1Q  *8$	((;;  %  --hA8$y'::)Ih,??&/mmoOD/C/C'3H  )Ih,??   %  --hA8$y'::''++"  w!GH!+)Ih,??'mmoG$$_S7%KL%/G'1H	  )Ih,??
   --hA$$= II@  }}HGG))(A6UWW\\* i
 %MMOMCGG))(A6 i 	9~X...	I
 s
s   $Oro   N)rq   rr   rs   rt   rJ  r  r  r  r  r  r  rv   ro   rw   rx   r
  r
    si      ,A ,A\ " " 	 	 @ @rw   r
  c                     ^  \ rS rSr% SrS\S'   SU 4S jjrSS jr\rS r	SS jr
SS	 jr\S
 5       rSS jrS rSS jrSS jrSS jrS rSS jrSSS jjrS S jrSrU =r$ )!r$  i  z*Pretend our storage has a different layoutr  r   c                   > [         TU ]  5         [        U R                  [        5      (       a0  [
        R                  U SU R                  R                  5       5        g g )Nr  )r  rC  r{   r  r  r   r:  r  r  s    rx   rC  ReinterpretView.__post_init__  sC    dii**tVTYY-B-B-DE +rw   c                P    U R                  U R                  U R                  /5      $ r   )rm  r  r   rB  s    rx   r  ReinterpretView.__str__  s&    		
 	
rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  ReinterpretView.get_name  r  rw   c                .    U R                   R                  $ r   )r   r   rB  s    rx   r   ReinterpretView.get_device  s    {{!!!rw   c                    g r   ro   rB  s    rx   rW  ReinterpretView.get_origin_node  r]  rw   c                .    U R                   R                  $ r   )r   r   rB  s    rx   r   ReinterpretView.dtype  s    {{   rw   c                @    [        U R                  R                  5      $ r   )r|   r   r   rB  s    rx   r   ReinterpretView.get_size  s    DKK$$%%rw   c                @    [        U R                  R                  5      $ r   )r|   r   r   rB  s    rx   r  ReinterpretView.get_stride  s    DKK&&''rw   c                   ^  SU 4S jjnU$ )Nc                Z  > TR                   R                  5       n[        R                  " TR	                  5       U" U 5      5      nTR                   R
                  TR                  R
                  :w  a6  [        R                  " UTR
                  TR                  R
                  5      $ U$ r   )r   r  rT   loadr  r   r  to_dtype_bitcast)r   r  
tmp_loaderr;  s      rx   r  +ReinterpretView.make_loader.<locals>.loader  sp    kk..0G$--/75>BJ{{  DIIOO3++J

DIIOOTT!!rw   r   r"  r   rU   ro   r;  r  s   ` rx   r  ReinterpretView.make_loader  s    	" rw   c                6    U R                   R                  5       $ r   )r   r  rB  s    rx   r  ReinterpretView.make_indexer      {{''))rw   c                    U R                   $ r   r   rB  s    rx   r   ReinterpretView.get_layout  r  rw   c                    g r   ro   rB  s    rx   r  ReinterpretView.freeze_layout  r  rw   c                    [        U R                  R                  5      [        U R                  R                  5      -  [        U R                  R                  5      -  $ r   )r&   r   r   r   r#  rB  s    rx   r  (ReinterpretView.get_unbacked_symbol_uses  sG    !$++"2"23#DKK$6$678#DKK$6$678	
rw   c           	     p   [         R                  R                  R                  U R                  U R
                  R                  U R
                  R                  U R
                  R                  Ub  UR                  O#[         R                  R                  R                  U R
                  R                  S9$ r  )rV   r   wrapper_codecodegen_reinterpret_viewr  r   r   r   r#  	writeliner   r  s     rx   r  !ReinterpretView.codegen_reference  s     ww##<<IIKKKKKK & 2F8L8L8V8V++## = 
 	
rw   c                    gr  ro   rB  s    rx   r  ReinterpretView.num_reads      rw   ro   r  r2  r(  r  r!  r,  r.  r  r~  r   r'  r@  )rq   rr   rs   rt   __doc__ru   rC  r  r  r  r   rW  rM  r   r   r  r  r  r   r  r  r  r  rv   r  r  s   @rx   r$  r$    sr    4NF

 H$" ! !&(	*

 rw   r$  c                  f    \ rS rSr% SrS\S'   \S 5       rSS jr\r	\
S 5       rSS jrSS	 jrS
rg)	DtypeViewi  z(Pretend our storage has a different typer  target_dtypec                    [        U5      (       aM  [        U5      u  p4[        UR                  UUR                  UR
                  UR                  5      n[        X5S9$ [        XS9$ )Nr  )r  r  )	r   r!  r"  r   r   r   r#  r$  r  )r  r   	new_dtyper'  r(  r*  s         rx   r  DtypeView.create  sa     ##"7":G$!!!!!!J #CCa88rw   c                P    U R                  U R                  U R                  /5      $ r   )rm  r  r  rB  s    rx   r  DtypeView.__str__  s     		4+<+<=>>rw   c                    U R                   $ r   )r  rB  s    rx   r   DtypeView.dtype  s       rw   c                6    U R                   R                  5       $ r   r  r   rB  s    rx   r   DtypeView.get_size  r  rw   c                L   ^ ^ T R                   R                  5       mUU 4S jnU$ )Nc                |   > [         R                  " T" U 5      TR                  TR                  R                  5      $ r   )rT   r  r  r  r   )r   r	  r;  s    rx   r  %DtypeView.make_loader.<locals>.loader  s*    ''c
D4E4EtyyWWrw   r  r  )r;  r  r	  s   ` @rx   r  DtypeView.make_loader  s"    		%%'	X rw   ro   Nr2  r!  r,  )rq   rr   rs   rt   r  ru   r  r  r  r  rM  r   r   r  rv   ro   rw   rx   r  r    sE    29 9? H! !$rw   r  c                  8    \ rS rSr\S 5       r\SS j5       rSrg)	SliceViewi  c                \  ^ ^^^^	^
 [         R                  R                  m
UR                  5       U   m[	        S X4T4 5       5      (       a!  [
        R                  m	[
        R                  mOT
R                  m	T
R                  mUU	U
4S jmUU U4S jnU" USTS5      nU" XCTT5      nX44$ )zb
Normalize start and end such that both are in the range
[0, x.get_size()[dim]] and start <= end.
c              3  8   #    U  H  n[        U5      v   M     g 7fr   r  r  r   s     rx   r  0SliceView.normalize_start_end.<locals>.<genexpr>%  s     H1GA$Q''1GrL  c                   > TR                  X5      (       a  U OT" X5      nTR                  X25      (       a  UnU$ T" X25      nU$ r   )statically_known_geqr  )r   lowerupperclamped_lowerclamped_fullmax_funcmin_funcr   s        rx   clamp,SliceView.normalize_start_end.<locals>.clamp,  s^    221<<(1BT 
 00FF  
   m3 
  rw   c                D   > U c  U$ TR                  U T5      n T" XU5      $ r   )r  )r  r  r  r  r  r  dim_sizes       rx   
clamp_wrap1SliceView.normalize_start_end.<locals>.clamp_wrap7  s,    {++C:CU++rw   r   )
rV   r   r   r   r  r   MinMaxevaluate_minevaluate_max)r  r   r  startendr  r  r  r  r  r   s   `     @@@@@rx   normalize_start_endSliceView.normalize_start_end  s     77##::<$H%h1GHHHyyHyyH,,H,,H		 	, 5!Xq1Xx8zrw   c           	       ^^^^ [         R                  " T5      m[        T[         R                  5      (       d  TS:  d   e TS:X  a  US:  a  TS:X  a  U$ [        UR                  5       5      mU(       a  U R                  UTTU5      u  mn[        UT-
  TS-
  -   T5      TT'   [        U5      (       av  [        U5      u  px[        UR                  5      n	U	T   T-  U	T'   [        UR                  UR                  TU	UR                  UR                  T   T-  -   5      n
[!        XzS9$ UUUU4S jn[#        UTUS9$ ! [         a     Nf = f)Nr   l    r1   r  c                   > [        U 5      [        T5      :X  d   SU  ST 35       e[        U 5      n U T   T-  T-   U T'   U $ )Nzwrong ndim rt  )r   r|   )r   r  rP  r  steps    rx   r   !SliceView.create.<locals>.reindexc  sP    u:X.P+eWAhZ0PP.KEsd*U2E#JLrw   r  )r   r  r{   r   	TypeErrorr|   r   r  r.   r   r!  r   r"  r   r   r#  r$  r  )r  r   r  r  r  r  r  r'  r(  r)  r*  r   rP  s     `` `      @rx   r  SliceView.createA  sQ   ||D!$

++tax77	zcY.419 

%
 00CDJE3 uq!94@ ##"7":Gj//0J(o4JsO$!!  !!J$5$5c$:U$BBJ #CC	 	 ah@@E  		s   E   
EEro   N)r1   T)rq   rr   rs   rt   r  r  r  rv   ro   rw   rx   r  r    s+    " "H (A (Arw   r  c                  R    \ rS rSr% S\S'   S\S'   SS jrSS jrSS jrSS	 jrS
r	g)BaseConstantim  r  r   r+  r   c                    gNro   ro   rB  s    rx   r   BaseConstant.get_sizer  s    rw   c                    U R                   $ r   r  rB  s    rx   r   BaseConstant.get_deviceu  r  rw   c                    g r   ro   rB  s    rx   rW  BaseConstant.get_origin_nodex  r]  rw   c                    [        5       $ r   r+   rB  s    rx   rN  BaseConstant.get_reads{  rv  rw   ro   Nr!  r(  r  r?  )
rq   rr   rs   rt   ru   r   r   rW  rN  rv   ro   rw   rx   r  r  m  s"    rw   r  c                  R    \ rS rSr% S\S'   S\S'   S\S'   SS jrSS	 jrSS
 jrSrg)Constanti  r   r   r  r   r+  r   c                   ^  SU 4S jjnU$ )Nc                Z   > [         R                  " TR                  TR                  5      $ r   )rT   r  r   r   r   r;  s    rx   r  $Constant.make_loader.<locals>.loader  s    <<

DJJ77rw   r  ro   r  s   ` rx   r  Constant.make_loader  s    	8 rw   c                    g r   ro   rB  s    rx   r  Constant.realize  r  rw   c                @    [        U R                  U R                  US9$ )N)r   r   r   )r(  r   r   r  s     rx   r  Constant.constant_to_device  s    djj

6JJrw   ro   Nr,  r%  rF  )	rq   rr   rs   rt   ru   r  r  r  rv   ro   rw   rx   r(  r(    s#    JKrw   r(  c                  H    \ rS rSr% S\S'   S\S'   S\S'   SS jrSS	 jrS
rg)IndexingConstanti  r   r   r  r   r+  r   c                   ^  SU 4S jjnU$ )Nc                Z   > [         R                  " TR                  TR                  5      $ r   )rT   r  r   r   r+  s    rx   r  ,IndexingConstant.make_loader.<locals>.loader  s    >>$**djj99rw   r  ro   r  s   ` rx   r  IndexingConstant.make_loader  s    	: rw   c                @    [        U R                  U R                  US9$ )N)r   r   r   )r3  r   r   r  s     rx   r  #IndexingConstant.constant_to_device  s    djj

6RRrw   ro   Nr,  rF  )rq   rr   rs   rt   ru   r  r  rv   ro   rw   rx   r3  r3    s    JSrw   r3  c           	     b    [        S [        U [        R                  U5      U5       5       5      $ )Nc              3  H   #    U  H  u  pnUS :H  =(       d    X:H  v   M     g7fr  ro   )r  leftrightr   s       rx   r  2is_contiguous_strides_for_shape.<locals>.<genexpr>  s.      "
D 		"T]""
s    ")r	  r   r   r   )r   r  s     rx   is_contiguous_strides_for_shaper?    s5      !$N55e<e"
  rw   c                <    [         R                  U R                  -  $ r   )r2   padding_alignment_bytesitemsizerp  s    rx   get_align_for_dtyperC    s    ))U^^;;rw   c                  ,    \ rS rSrSrSS jrSS jrSrg)	r   i  ztAbstract base for Layout, MultiOutputLayout, NoneLayout.
Represents the memory layout of the output of an Operation.c                >    [        [        U 5      R                  5      er   r  rB  s    rx   r   OutputSpec.get_device  r  rw   c                >    [        [        U 5      R                  5      er   r  rB  s    rx   storage_sizeOutputSpec.storage_size  r  rw   ro   Nr(  r@  )rq   rr   rs   rt   r  r   rH  rv   ro   rw   rx   r   r     s    C77rw   r   c                      \ rS rSrS\" S5      4           SS jjrSS jr\rSS jrSS jr	\
      SS j5       rSS	 jrSS
 jrS r\
S 5       rS rS rS rSS jrSS jrSS jrSrg)r  i  Nr   c                    Uc  [         R                  U5      nXl        X l        [	        U5      [	        U5      :X  d   SU SU 35       e[        S U 5       5      (       d   eX0l        X@l        XPl        g )Nr  	, stride=c              3  N   #    U  H  n[        U[        [        45      v   M     g 7fr   )r{   r   r   r  s     rx   r  "Layout.__init__.<locals>.<genexpr>  s     <t!:a$--t   #%)	r   r   r   r   r   r	  r   r   r#  )r;  r   r   r   r   r#  s         rx   rC  Layout.__init__  sq     >#66t<F
4yCK'H5ix)HH'<t<<<<< $	"("rw   c                \   SnU R                   S:w  a  SU R                    3nU R                  R                  c  SOSU R                  R                   3n[        U 5      R                   SU R                  R                   U SU R
                   SU R                   SU R                   U S	3$ )
Nr_  r   z	, offset=:z('z', z, size=rL  r   )r#  r   r   r   rq   r   r   r   )r;  r#  device_index_strs      rx   r  Layout.__str__  s    ;;! .F!%!2!2!:2!DKKDUDUCV@WDz""#2dkk&6&6%78H7ITZZL YII;i}VHA?	
rw   c                    U R                   $ r   r  rB  s    rx   r   Layout.get_device  r  rw   c                B    [        U R                  U R                  5      $ r   )r?  r   r   rB  s    rx   r  Layout.is_contiguous  s    .t{{DIIFFrw   c                    [        U 5      nUS;  d	  U S   S:X  a  g[        U[        U 5      U 5       H  u  p4nUS:w  d  M  X4:w  d  M    g   g)N)r      r1   FT)r   r   r"   )r  r&  ndimr<  r=  r   s         rx   is_channels_last_contiguous"Layout.is_channels_last_contiguous  sX     5zvqQ!$3E:E"
D qyT]	"

 rw   c                    [        U R                  [        [        R	                  [        [        U R                  5      5      5      5      U R                  5       H  u  pnUS:w  d  M  X:w  d  M    g   g)Nr1   FT)r   r   reversedr   r   r|   r   )r;  r<  r=  r   s       rx   is_transposedLayout.is_transposed  sZ    !$KK^66tHTYY<O7PQRII"
D
 qyT]"
 rw   c           	        [        U R                  5      [        U5      :X  d   e[        U R                  5       VVs/ s H5  u  p#[        R
                  R                  R                  USS9S:w  d  M3  UPM7     nnnU Vs/ s H  o R                  U   PM     nnU Vs/ s H  o!U   PM	     nnS nU" U5      nS/[        U5      -  n[        [        U5      5       H  nXR   XqU   '   M     [        [        U5      S-
  5       H_  nXr   XrS-      :  n[        U[        5      (       d2  [        R
                  R                  R                  Xr   XrS-      :  SS9nU(       d  M_    g   gs  snnf s  snf s  snf )	Nr   rM  r1   c                d    [        U 5      nU  Vs/ s H  o!R                  U5      PM     sn$ s  snf r   )r  r   )arr
sorted_arrelements      rx   sorted_indices0Layout.is_stride_ordered.<locals>.sorted_indices  s,    J=@AS'$$W-SAAAs   -r  TrK  F)r   r   r   r   rV   r   r   r   r   r{   rm   
_shape_envrO  )	r;  r   r   r  non_1_indicesr   rg  stride_orderedexprs	            rx   r  Layout.is_stride_ordered  sl   4;;3u:---
 $DII.
.ww))#):a? . 	 
 +88-Q++a.-8#01=aq=1	B
 u% E
*s5z"A'-yN8$ # s5zA~&A!$~!e'<<DdD))ww))77"%1u(==d 8  t ' ;
 91s   2E'3E' E-E2c                    S/[        [        [        S[        U R                  5      S-
  5      5      5      -   n[        U5      /U-   nU R                  U5      $ Nr   r1   )r|   r_  r   r   r   r  r  s     rx   is_channels_last_stride_ordered&Layout.is_channels_last_stride_ordered  sN    d8E!S-=-A$BCDDUu$%%e,,rw   c                   [        U5      n[        U 5      S:X  a  U $ [        R                  (       d  [        R                  X5      (       a  U $ [        R                  " 5       n[        US5      (       a#  UR                  R                  SS5      (       a  U $ [        S [        R                  " X5       5       5      (       d  U $ [        U 5      n[        U5      n[!        [        U 5      5       Vs/ s H  nSPM     nnSXS   '   Sn	[#        USS SS9 HE  u  pXjS-
     nX   X   -  nU[        R$                  :  a  X-  S:w  a  ['        X5      U-  nS	n	XU'   MG     U	(       d  U $ [(        =R*                  S-  sl        U$ s  snf )
zv
The padding does not change stride order but makes sure all strides larger
than the threshold are multiple of align.
r   metadislike_paddingFc              3  b   #    U  H%  n[        U[        [        R                  45      v   M'     g 7fr   )r{   r   r   r   r  s     rx   r  &Layout._pad_strides.<locals>.<genexpr>>  s*      
6 q3.//6s   -/r1   N)r  T)rC  r   r2   pad_channels_lastr  r\  rV   get_current_noderZ  rs  getr	  r  chainr   r   r   r   padding_stride_thresholdrG   r   num_comprehensive_padding)
in_stridesr   r   aligncurrent_fx_noder  r   r   new_stridespaddedrankr   prev_idxr   s                 rx   _pad_stridesLayout._pad_strides$  s    $E*z?a''F,N,N-
 -
 ,,.?F++0D0D0H0Hu1
 1
   
__Z6
 
 
 '
3,\:
"'J"89"8Qq"89 &'qM"":ab>;ID!(+H *T^;F777FNa<O /%7% <  ))Q.)- :s   0Fc                    [        U [        5      (       d   eU R                  c   eU R                  U R                  U R                  U R
                  5      U l        g r   )r{   r   r   r  r   r   rB  s    rx   r  Layout.pad_strides_  sG    $////{{&&&''TYY

Krw   c                P    [         R                  =(       a    [        U [        5      $ r   )r2   comprehensive_paddingr{   r   rB  s    rx   r  Layout.should_pad_stridesd  s    ++P
40PPrw   c                    [        U [        5      (       a  U $ U R                  5       (       a  U R                  5         [        U R                  U R
                  U R                  U R                  U R                  5      $ r   )	r{   r"  r  r  r   r   r   r   r#  rB  s    rx   as_fixedLayout.as_fixedg  s_    dK((K""$$KKJJIIKKKK
 	
rw   c                    [         R                  (       d   S[        U 5      R                   S35       eU R	                  5       R                  5       $ )Nzconvert z to FixedLayout first)r   r  r   rq   r  r  rB  s    rx   r  Layout.make_indexeru  sG    ,, 	
tDz**++@A	
, }}++--rw   c                4   U R                   UR                   :H  =(       ay    U R                  UR                  :H  =(       aY    U R                  UR                  :H  =(       a9    U R                  UR                  :H  =(       a    U R                  UR                  :H  $ r   r   r   r   r   r#  )r;  others     rx   __eq__Layout.__eq__{  so    KK5<<' ,

ekk),		UZZ', u||+, u||+	
rw   c                X    [        U R                  U R                  U R                  5      $ r   )r   r   r   r#  rB  s    rx   rH  Layout.storage_size  s    .tyy$++t{{SSrw   )r   r   r#  r   r   )r   r+  r   r  r   r5  r   zOptional[list[Expr]]r#  r   r   r   r2  r*  r   )r  r1  r&  r1  r   rm   r.  r   z
sympy.Expr)rq   rr   rs   rt   r   rC  r  r  r   r  rJ  r\  r`  r  rp  r  r  r  r  r  r  rH  rv   ro   rw   rx   r  r    s     (,qz## # 	#
 %# # 
#$	
 HG !,>	 !F- 8 8tL
Q
.
Trw   r  c                  "    \ rS rSrSrSS jrSrg)r"  i  z A Tensor layout we cannot changec                   ^  U 4S jnU$ )z1A closure containing math to read a given elementc                $  > [        U 5      [        TR                  5      :X  d   e[        U 5      [        TR                  5      :X  d   eTR                  n[	        U TR                  TR                  5       H  u  p#nUS:w  d  M  XU-  -   nM     U$ r  )r   r   r   r#  r   )r   r  r   r   szr;  s        rx   r  )FixedLayout.make_indexer.<locals>.indexer  s{    u:T[[!1111u:TYY///[[F#&udkk499#ER7#Fl2F $F Mrw   ro   r;  r  s   ` rx   r  FixedLayout.make_indexer  s    	 rw   ro   Nr.  )rq   rr   rs   rt   r  r  rv   ro   rw   rx   r"  r"    s
    *rw   r"  c                     ^  \ rS rSrSrSr\S 5       r\S 5       r\S 5       r	\S 5       r
\S 5       rSS	 jrSS
 jrS rS rSSU 4S jjjrSrU =r$ )r   i  z(A Tensor layout we are allowed to changeFc                    [        U 5      S:X  a  / $ [        R                  R                  /n[	        U SS  5       H  nUR                  X!S   -  5        M     [        [	        U5      5      $ )Nr   r1   r  )r   r   r  r  r_  r  r|   )sizesreversed_stridesr   s      rx   r   !FlexibleLayout.contiguous_strides  s^    u:?I!GGKK=U12Y'D##DB+?$?@ (H-.//rw   c                    [        [        [        U 5      5      5      [        U5      :X  d   X45       e[        R                  R
                  nS/[        U5      -  nU H  nX#U'   X U   -  nM     U$ )z
Create a stride based on the order the dimensions should be filled in.

In this format, channels last would be:
    [1, 3, 2, 0]
N)r,   r   r   r   r  r  )r  r   next_strider&  r   s        rx   fill_orderedFlexibleLayout.fill_ordered  sm     %E
+,
50AAQE>QAggkk&3u:%A$AJ%a0K  rw   c                    [        [        [        U 5      5      5      [        U5      :X  d   e[        U5      n[        R                  X5      $ )zz
Create a stride based on the sorted order of a permuted range.

In this format, channels last would be:
    [3, 0, 2, 1]
)r,   r   r   r   r   r  )r  r   r   s      rx   rk  FlexibleLayout.stride_ordered  s@     %E
+,
50AAAA,U3
**5==rw   c                D   U[         R                  :X  a  [        R                  U [        5      $ U[         R
                  :X  a  [        R                  U [        5      $ U[         R                  :X  a  [        R                  U 5      $ [        R                  SU5        [        e)a9  
Create a stride based on a memory format.

Memory format is translasted into a stride order,
so channels_last is the same as:
    FlexibleLayout.stride_ordered(sizes, [3, 0, 2, 1])

This interface does not support memory_format `torch.preserve_format`
which should be used to deduce a format from another source
z>stride_ordered_for_memory_format, unsuppored memory_format: %s)r   channels_lastr   rk  NHWC_STRIDE_ORDERchannels_last_3dNHWDC_STRIDE_ORDERcontiguous_formatr   r  r  rt  )r  memory_formats     rx    stride_ordered_for_memory_format/FlexibleLayout.stride_ordered_for_memory_format  s     E///!008IJJe444!008JKKe555!44U;;IIP &%rw   c                (   [        U 5      [        U5      :X  d   eU Vs/ s H,  n[        R                  R                  R	                  U5      PM.     nn[        [        [        U5      5      UR                  S9n[        R                  X5      $ s  snf )z
Create a stride that has the same stride order as given stride

For example, if given stride is [1000, 1, 100, 10],
the fill order should be [1, 3, 2, 0]
r|  )
r   rV   r   r   r   r  r   __getitem__r   r  )r  r   r   r   s       rx   same_orderedFlexibleLayout.same_ordered  su     5zS[(((9?@A!''"",,Q/@E#f+.F4F4FG
**5== As   3Bc                ,   U R                  U R                  U5      nU R                  5       (       a-  U(       a&  U R                  X0R                  U R                  5      n[        U R                  U R                  U R                  UU R                  5      $ r   )rk  r   r  r  r   r"  r   r#  )r;  r   r  r)  s       rx   as_stride_orderFlexibleLayout.as_stride_order  so    ((E:
""$$**:yy$**MJKKJJIIKK
 	
rw   c                    UnU R                  5       (       a-  U(       a&  U R                  X0R                  U R                  5      n[	        U R
                  U R                  U R                  UU R                  5      $ r   )r  r  r   r   r"  r   r#  )r;  r  r  r)  s       rx   as_exact_stridesFlexibleLayout.as_exact_strides  s^    "
""$$**:yy$**MJKKJJIIKK
 	
rw   c                   U R                  U R                  U5      nU R                  5       (       a&  U R                  X R                  U R                  5      n[        U R                  U R                  U R                  UU R                  5      $ r   )r  r   r  r  r   r"  r   r#  )r;  r   r)  s      rx   as_fill_orderFlexibleLayout.as_fill_order  sm    &&tyy%8
""$$**:yy$**MJKKJJIIKK
 	
rw   c                   U R                  U R                  U5      nU R                  5       (       a&  U R                  X R                  U R                  5      n[        U R                  U R                  U R                  UU R                  5      $ r   )r  r   r  r  r   r"  r   r#  )r;  r   r)  s      rx   as_same_orderFlexibleLayout.as_same_order  sm    &&tyy&9
""$$**:yy$**MJKKJJIIKK
 	
rw   c                   > U(       a  [         R                  X45      nO[         R                  U5      n[        TU ]  XX55        g r   )r   r  r   r  rC  )r;  r   r   r   r  r&  r  s         rx   rC  FlexibleLayout.__init__  s5    $11$EG$77=G6rw   ro   r7  r   r  )rq   rr   rs   rt   r  r  rJ  r   r  rk  r  r  r  r  r  r  rC  rv   r  r  s   @rx   r   r     s    2N 0 0    	> 	> & &0 
> 
>





7 7rw   r   c                  @   ^  \ rS rSrSrSU 4S jjrSS jrS rSrU =r	$ )	NonOwningLayouti&  z,Is a view into the storage of another tensorc                   > UR                  5       n[        TU ]	  UR                  UR                  UR
                  UR                  5        Xl        g r   )r   r  rC  r   r   r   r   view)r;  r  r   r  s      rx   rC  NonOwningLayout.__init__)  s?    "MMLLKKMM		
 	rw   c                >    U R                  5       R                  5       $ r   )r  r  rB  s    rx   r  NonOwningLayout.make_indexer3  s    }}++--rw   c                    U R                   R                  5       R                  nUS:X  a  gSSKJn  [
        R                  R                  R                  X5      $ )Nr   Tr1   )	ALIGNMENT)	r  r   r#  utilsr  rV   r   r   statically_known_multiple_of)r;  r#  r  s      rx   maybe_guard_aligned#NonOwningLayout.maybe_guard_aligned6  sB    %%'..Q;$ww<<VOOrw   )r  )r  zUnion[BaseView, TensorBox]r   r   r.  )
rq   rr   rs   rt   r  rC  r  r  rv   r  r  s   @rx   r  r  &  s    6.P Prw   r  c                      \ rS rSrSrSrg)CommBufferTypei?  symm_memro   N)rq   rr   rs   rt   SYMM_MEMrv   ro   rw   rx   r  r  ?  s    Hrw   r  c                  R   ^  \ rS rSr% SrS\S'   S\S'         S	U 4S jjrSrU =r$ )
CommBufferLayoutiC  a\  
A layout that signifies the buffer is a comm buffer.
In terms of striding, the layout is identical to `FixedLayout`.

Buffers with this layout do not participate in in-place reuse - it can be
neither the source nor the target for in-place reuse.

For detailed motivation and usage of this layout, see
NOTE [lowering-time collective optimization].
r  comm_buffer_typer   
group_namec                  > [        U[        5      (       d  [        SU S35      eUR                  5       n[        TU ]  UR                  UR                  UR                  UR                  UR                  S9  X l        X0l        g )NzJA `CommBufferLayout` can only be initialized with a `FlexibleLayout` (got z).r  )r{   r   r{  r  r  rC  r   r   r   r   r#  r  r  )r;  r   r  r  fixedr  s        rx   rC  CommBufferLayout.__init__R  s     &.11 ++1("6 
 !<<++<<<< 	 	
 !1$rw   )r  r  )r   r   r  r  r  r   )	rq   rr   rs   rt   r  ru   rC  rv   r  r  s   @rx   r  r  C  s;    	 %$O%% )% 	% %rw   r  c                      \ rS rSr% S\S'   \R                  " S S9rS\S'   \R                  " S S9rS\S	'   SS
 jr	S r
SS jrSrg)
NoneLayoutij  r)  r   c                     S/$ r  ro   ro   rw   rx   rz  NoneLayout.<lambda>u  s    rw   default_factoryr9  r   c                     S/$ r  ro   ro   rw   rx   rz  r  v  s    1#rw   r   c                    gr  ro   rB  s    rx   rH  NoneLayout.storage_sizex  r  rw   c                    U $ r   ro   rB  s    rx   r  NoneLayout.as_fixed{      rw   c                    U R                   $ r   r  rB  s    rx   r   NoneLayout.get_device~  r  rw   ro   Nr@  r(  )rq   rr   rs   rt   ru   rH  rI  r   r   rH  r  r   rv   ro   rw   rx   r  r  j  sC     #"!''DD)D#))+FFIFrw   r  c                     ^  \ rS rSrSU 4S jjr\SS j5       r\R                  SS j5       rSS jrSS jr	S r
\SS j5       rS	 rSS
 jrSrU =r$ )MutationLayoutSHOULDREMOVEi  c                  > [         TU ]  UR                  5       UR                  5       UR	                  5       S 5        Xl        U R                  5       R                  5       n[        R                  R                  U5        g r   )r  rC  r  r   r   r]  
get_bufferr  rV   r   mark_buffer_mutated)r;  r]  r   r  s      rx   rC  #MutationLayoutSHOULDREMOVE.__init__  sc    &&(OO		
  ))+	##D)rw   c                6    U R                  5       R                  $ r   )real_layoutr   rB  s    rx   r   !MutationLayoutSHOULDREMOVE.stride  s    !(((rw   c                    g r   ro   )r;  r   s     rx   r   r    s    rw   c                >    U R                  5       R                  5       $ r   )r  rH  rB  s    rx   rH  'MutationLayoutSHOULDREMOVE.storage_size  s    !..00rw   c                p   ^ U4S jmT" U R                   5      n[        U[        5      (       d   S5       eU$ )Nc                   > [        U [        5      (       a  T" U R                  5      $ [        U [        5      (       a  T" U R	                  5       5      $ [        U [
        5      (       a  T" U R                  5      $ U $ r   )r{   r  r]  r  r  
MutableBoxr  )r]  unwrap_viewss    rx   r  ;MutationLayoutSHOULDREMOVE.get_buffer.<locals>.unwrap_views  sb    &"<==#FMM22&(++#F$6$6$899&*--#FKK00Mrw   z1MutationLayoutSHOULDREMOVE must refer to a buffer)r]  r{   r  )r;  r  r  s     @rx   r  %MutationLayoutSHOULDREMOVE.get_buffer  s<    	 dkk*&&)) 	
?	
) rw   c                6    U R                  5       R                  $ r   )r  r   rB  s    rx   r  &MutationLayoutSHOULDREMOVE.real_layout       '''rw   c                   UR                  5         [        R                  R                  UR	                  5       5        [        U[        5      (       a  UR                  nUR                  5         U(       d  [        R                  UR                  5       UR                  5       UR                  5       [        UR                  5       UR                  5       5       VVs/ s H.  u  pE[        R                  R                   R#                  XE5      PM0     snnS9R                  nUR                  5         [        UR                  R$                  [&        5      (       d   e[)        U5      UR                  l        UR                  $ s  snnf )Nr  )r  rV   r   r  r  r{   rb   r  r  r  r  r   r   r  r   r   r   r  r   r   r  )r  srcdstunsafe_aliasr'  r(  s         rx   realize_into'MutationLayoutSHOULDREMOVE.realize_into  s    	
##CLLN3c9%%((C 	""~~'mmo* !$CLLNCLLN C C GG$$11!7 C	 #  d  	#((//>::::4S9xxs   5Fc                    U $ r   ro   rB  s    rx   r  #MutationLayoutSHOULDREMOVE.as_fixed  r  rw   c                6    U R                   R                  5       $ r   )r]  r  rB  s    rx   r  'MutationLayoutSHOULDREMOVE.make_indexer  r  rw   )r]  )r]  rc   r   r   r   r5  )r   r   r   r   r  )r   r  r7  r.  )rq   rr   rs   rt   rC  rM  r   setterrH  r  r  r  r  r  r  rv   r  r  s   @rx   r  r    sg    	* ) ) ]] 1 (    D* *rw   r  c                  Z  ^  \ rS rSr% S\S'   S\S'   S#U 4S jjrS$S jrS%S jrS&S	 jrS'S
 jr	\
S(S j5       rS)S jrS*S jrS+S jrS,S jrS-S jrS rS rS.S#S jjrS#S jrS#S jr S. S#S jjrS rS/S jrS0S1S jjrS rS2S jrS2S jrS3S jrS4S jrS4S jrS5S  jr S6S! jr!S"r"U =r#$ )7r  i  r&  r   r   r   c                F   > [         TU ]  5         U R                  SS 5        g rY  )r  rC  r=  r  s    rx   rC  Buffer.__post_init__  s    t4rw   c                >    U R                  5       R                  5       $ r   )r   r  rB  s    rx   r  Buffer.make_indexer  s     --//rw   c                J    U R                   (       d   U 5       eU R                   $ r   rG  rB  s    rx   r  Buffer.get_name  s    yy$yyyrw   c                >    U R                  5       R                  5       $ r   )r  r   rB  s    rx   r   Buffer.get_device  s    ##%0022rw   c                    g r   ro   rB  s    rx   r[  Buffer.get_defining_op  r]  rw   c                6    U R                  5       R                  $ r   )r   r   rB  s    rx   r   Buffer.dtype  s     &&&rw   c                :    / U R                  5       R                  Q$ r   )r   r   rB  s    rx   r   Buffer.get_size  s    ("''((rw   c                :    / U R                  5       R                  Q$ r   )r   r   rB  s    rx   r  Buffer.get_stride  s    *"))**rw   c                6    U R                  5       R                  $ r   )r   r#  rB  s    rx   
get_offsetBuffer.get_offset  r  rw   c                    [        U R                  [        5      (       a  U R                  $ [        [	        U R                  5      R
                  5      er   )r{   r   r  rt  r   rq   rB  s    rx   r   Buffer.get_layout  s7    dkk6**;;!$t{{"3"<"<==rw   c                    U R                   $ r   r  rB  s    rx   r  Buffer.get_output_spec  r  rw   c                "    U R                  5       $ r   )r  rB  s    rx   r  Buffer.get_storage_numel  s    ~~rw   c                    [        U R                  [        5      (       a@  [        U R                  [        5      (       d   U R                  R	                  5       U l        g g g r   )r{   r   r  r  r  rB  s    rx   r  Buffer.freeze_layout	  sF    dkk6**:KK4
 4
 ++..0DK4
*rw   c                    [        U R                  [        5      (       d   eU R                  R                  XS9U l        g Nr  )r{   r   r   r  r  s      rx   r  &Buffer.freeze_layout_with_stride_order  s2    $++~6666kk11%1Urw   c                    [        U R                  [        5      (       d   eU R                  R                  U5      U l        g r   )r{   r   r   r  r  s     rx   r  $Buffer.freeze_layout_with_fill_order  s/    $++~6666kk//6rw   c                    [        U R                  [        5      (       d   eU R                  R                  U5      U l        g r   )r{   r   r   r  r  s     rx   r  $Buffer.freeze_layout_with_same_order  s/    $++~6666kk//7rw   c                    [        U R                  [        5      (       d   eU R                  R                  XS9U l        g r)  )r{   r   r   r  r  s      rx   r  'Buffer.freeze_layout_with_exact_strides  s9     $++~6666kk22 3 
rw   c                    [         R                  R                  R                  [        R
                  " U R                  5       S5      5      $ r  r  rB  s    rx   r  Buffer.is_zero_elements#  r  rw   c                v   ^  T R                  5       (       a  [        [        T R                  5       S9$ U 4S jnU$ )Nrp  c                   > TR                  5       n[        R                  " TR                  =(       d    SU" U 5      5      $ r  )r  rT   r  r   r   r  r;  s     rx   r  "Buffer.make_loader.<locals>.loader+  s/    '')G88DII2GENCCrw   )r  r
   r  r   r  s   ` rx   r  Buffer.make_loader&  s3      ""=0@AA	D rw   c                "    U R                  5       $ r   r  r  s     rx   r  Buffer.codegen_reference1  r"  rw   c                    g r   ro   rB  s    rx   r  Buffer.decide_layout4  r  rw   c                    [        U R                  [        5      (       a%  U R                  R                  R	                  5       /$ gr  )r{   r   r  r  r  rB  s    rx   r  #Buffer.get_inputs_that_alias_output7  s2    dkk?33KK$$--/00rw   c                    [        U R                  [        5      (       a%  U R                  R                  R	                  5       /$ gr  )r{   r   r  r]  r  rB  s    rx   r  Buffer.get_mutation_names<  s3    dkk#=>>KK&&//122rw   c                6    [        U R                  5       /5      $ r   )r,   r  rB  s    rx   rO  Buffer.get_read_namesA  s    4==?+,,rw   c                    [        5       $ r   r+   rB  s    rx   r  Buffer.get_unbacked_symbol_usesD  rv  rw   c                    [        5       $ r   r+   rB  s    rx   rt  Buffer.get_unbacked_symbol_defsG  rv  rw   c                    g r   ro   rB  s    rx   r  Buffer.realizeJ  r  rw   c                    gr  ro   rB  s    rx   should_allocateBuffer.should_allocateM  s    rw   r  r  r.  r2  r(  r  r  r!  r
  r$  r  r  r7  r,  r   r'  rG  r  r~  r%  r   )$rq   rr   rs   rt   ru   rC  r  r  r   r[  rM  r   r   r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  rO  r  rt  r  rJ  rv   r  r  s   @rx   r  r    s     
503 ' ')+(>
 1V78
 ,1
	
W	

- rw   r  c                  J    \ rS rSrSS jrSS jr\R                  rS	S jrSr	g)
OperationBufferiR  c                    U /$ r   ro   rB  s    rx   rq  OperationBuffer.get_outputsU  s	    vrw   c                    U $ r   ro   rB  s    rx   r[  OperationBuffer.get_defining_opX  r  rw   c                X    [         R                  U 5        [        R                  U 5        g r   )r  rC  rO  rB  s    rx   rC  OperationBuffer.__post_init__^  s    T"%rw   ro   Nr|  r   rO  r  )
rq   rr   rs   rt   rq  r[  rO  r  rC  rv   ro   rw   rx   rM  rM  R  s     #55&rw   rM  c                      \ rS rSrSS jrSrg)InputBufferic  c                    gr  ro   rB  s    rx   r  InputBuffer.num_readsd  r  rw   ro   Nr@  )rq   rr   rs   rt   r  rv   ro   rw   rx   rV  rV  c  s    rw   rV  c                      \ rS rSrSrSrg)DonatedBufferih  aA  
Represents a donated buffer which is a saved tensor that is not alias to any
fwd inputs, fwd user outputs, and bwd outputs. We generally cannot inplace
reuse the input tensor memory during backward since it might be used in another
function. However, donated buffer can be inplace reused during backward
to save memory.
ro   N)rq   rr   rs   rt   r  rv   ro   rw   rx   rZ  rZ  h  s    rw   rZ  c                  8    \ rS rSr% SrS\S'   SS jrS	S jrSrg)
r  ir  Nr)  r   c                   ^  SU 4S jjnU$ )Nc                   > TR                  5       R                  5       n[        R                  " [        R
                  R                  TR                  5       TR                  5      U" U 5      5      $ r   )	r   r  rT   r  rV   r   constant_namer  r   r5  s     rx   r  *ConstantBuffer.make_loader.<locals>.loaderv  sP    oo'446G88%%dmmot7K7KL rw   r  ro   r  s   ` rx   r  ConstantBuffer.make_loaderu  s    	 rw   c                    [        [        R                  R                  U R	                  5       U5      U R
                  S9$ N)r   r   )r  rV   r   r^  r  r   r  s     rx   r  !ConstantBuffer.constant_to_device  s/    &&t}}?
 	
rw   ro   r,  rF  )	rq   rr   rs   rt   r   ru   r  r  rv   ro   rw   rx   r  r  r  s    .2O+2
rw   r  c                  J    \ rS rSrS	S jrS
S jrSSS jjrSS jrSS jrSr	g)NoneAsConstantBufferi  c                    [        5       $ r   r+   rB  s    rx   rN  NoneAsConstantBuffer.get_reads  rv  rw   c                    [        5       $ r   r+   rB  s    rx   r  -NoneAsConstantBuffer.get_unbacked_symbol_uses  rv  rw   Nc                J    [         R                  R                  R                  $ r   )rV   r   r  none_strr  s     rx   r  &NoneAsConstantBuffer.codegen_reference  s    ww##,,,rw   c                    [        S S9$ Nr  )r  rB  s    rx   r  $NoneAsConstantBuffer.get_output_spec  s    &&rw   c                    gr  ro   rB  s    rx   r  &NoneAsConstantBuffer.has_tensor_output  r  rw   ro   r?  r~  r   r'  r  r   )
rq   rr   rs   rt   rN  r  r  r  r  rv   ro   rw   rx   re  re    s    -'rw   re  c                  B    \ rS rSr% S\S'   S	S jrS
SS jjrSS jrSrg)r   i  r   rl  c                ,    [        U R                  5      $ r   )r&   rl  rB  s    rx   r  .ShapeAsConstantBuffer.get_unbacked_symbol_uses  s    $TYY//rw   Nc                h    [         R                  R                  R                  U R                  5      $ r   )rV   r   r  codegen_sizevarrl  r  s     rx   r  'ShapeAsConstantBuffer.codegen_reference  s!    ww##33DII>>rw   c                    gr  ro   rB  s    rx   r  'ShapeAsConstantBuffer.has_tensor_output  r  rw   ro   r~  r   r'  r   )	rq   rr   rs   rt   ru   r  r  r  rv   ro   rw   rx   r   r     s    
J0?rw   r   c                    ^  \ rS rSr% S\S'   SS jrSS jrSS jrSS jrSS jr	SS	 jr
SU 4S
 jjrSS jrSS jrS S jr\  S!S j5       r  S"     S#S jjr\ S$S j5       rS%S jrSS jrS&S jrS&S jrS'S jrSrU =r$ )(r  i  r  r  c                    U R                   b  U R                   $ [        U R                  S5      (       a  U R                  R                   $ g)z}
Returns self.name if it exists, otherwise returns the name of the data node if that exists.
If neither exist, returns None.
Nr   )r   rZ  r  rB  s    rx   get_computed_buffer_name'ComputedBuffer.get_computed_buffer_name  s:    
 99 99499f%%99>>!rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  ComputedBuffer.num_reads  r:  rw   c                6    U R                   R                  5       $ r   r  rN  rB  s    rx   rN  ComputedBuffer.get_reads  r:  rw   c                6    U R                   R                  5       $ r   r@  rB  s    rx   rO  ComputedBuffer.get_read_names  rB  rw   c                   [         R                  " [        SS5         U R                  R	                  5       (       aT  [        U R                  5       U R                  R                  5       U R                  R                  5       5      sS S S 5        $ [        U R                  5       U R                  R                  5       5      sS S S 5        $ ! , (       d  f       g = fr  )
r   r   r   r  r  r:   get_store_functionr  r  r   rB  s    rx   r  ComputedBuffer.get_read_writes  s    \\.*:DAyy++--*++-II002II002 BA +++-II&&( BAAs   A*C1C
Cc                    [        U R                  5       5      [        U R                  5       5      -  [        U R                  5       5      -  U R                  R                  5       -  $ r   )r&   r   r  r  r  r  rB  s    rx   r  'ComputedBuffer.get_unbacked_symbol_uses  sV    & "$--/2#DOO$567#DOO$567 ii0023	
rw   c                   > U R                  5       (       dV  U R                  [        R                  R                  ;  a.  U R                  5       S:X  a  U R                  R                  5       $ [        TU ]  5       $ r  )	r  r   rV   r   mutated_buffersr  r  r  r  r  s    rx   r  ComputedBuffer.make_loader  sZ    ''))		!8!88 A% 99((**w"$$rw   c                   U R                  5       R                  5       R                  5       n[        U R                  [
        [        [        45      (       a+  [        U R                  R                  U R                  U5      $ [        U R                  [        5      (       d   e[        U R                  R                  U R                  U5      $ r   )r   r  r  r{   r  rH  ru  r  r
   rY  r   r  r  r  s     rx   r  !ComputedBuffer.get_store_function  s    //#,,.;;=dii)T4!89949944diiIIdii333349911499gFFrw   c                   [        U R                  [        5      (       Ga  [        R                  " U R
                  R                  5       U R
                  R                  5       5      u  u  pnU R                  5       R                  n[        S U 5       5      (       d   eU VVs/ s Hk  n[        U[        R                  5      (       d  M$  [        UR                  U Vs0 s H%  ofS:w  d  M
  U[        R                  R                   _M'     sn5      PMm     nnnU(       a  [        U R
                  ["        [$        45      (       a  U R
                  R'                  X5      nOUnU Vs/ s H,  n[(        R*                  R,                  R/                  X5      PM.     n	nSSKJn
  U
" XR5                  5       5      $ gs  snf s  snnf s  snf )aD  
If our layout is still flexible, try to determine the stride order based on stride orders of reads.

TODO(jansel): A better algorithm here would look at downstream consumers of this
              value and try to do global graph-level layout optimization.
              This is also something just begging to be autotuned.
c              3  v   #    U  H/  n[        U[        R                  [        R                  45      v   M1     g 7fr   )r{   r3   StarDep	MemoryDepr  s     rx   r  0ComputedBuffer.get_fill_order.<locals>.<genexpr>  s2      A 1|33\5K5KLMMs   79r   r1   pick_loop_orderN)r{   r   r   r3   r  r  r  r  r  r  r	  r  rS   r   r   r  r  ru  r  r   rV   r   r   r  	schedulerr  r   )r;  
index_varsrZ  r   r  r  vr  rl  stride_lengthsr  s              rx   r   ComputedBuffer.get_fill_order  sx    dkk>22.:.M.M		,,.		0L0L0N/+(Z! ((*00E       Aa!7!78 Y
177n$WnUVPV_Q_n$WX   dii$66"ii//
KG(GMR"MRTAGG$$11$@U  " 7&~}}GG# %X"s*   %#F?F? 	F:-F:F?(3G:F?c                    [        U R                  [        5      (       a:  U R                  5       nU(       a  U R	                  U5        g U R                  5         g g r   )r{   r   r   r   r  r  r  s     rx   r  ComputedBuffer.decide_layout  sC    dkk>22'')E2259""$ 3rw   c                   [         R                  " U R                  R                  5       U R                  R	                  5       SS9u  p[
        R                  " [        SU R                  5       5         [        U R                  5       U R                  5       (       a  UOUS S U/UQ76 nS S S 5        / n/ n/ n/ nUR                  5        Hf  u  pXS   ;   a-  U(       a   eUR                  U5        UR                  U	5        M:  XS   ;   d   eUR                  U5        UR                  U	5        Mh     Xg4WXE44$ ! , (       d  f       N= f)Nqr`   r   r1   r   )r3   r  r  r  r  r   r   r  r   r<   r  r  itemsr  )
r;  r   
var_rangesr  r  reduce_vars
index_sizereduce_sizer  r   s
             rx   get_default_sizes_body%ComputedBuffer.get_default_sizes_body%  s6    (::II((*DII,H,H,JSV
 \\.*;T__=NO'')0022Ra 	D P 
!#
$$&DAG|&&!!!$!!!$G|#|""1%""1% ' ($0III) POs   37D;;
E	c                  ^ ^^ T R                  5       u  u  p4nu  pgU(       a  U" X44XVU45      u  u  p4nu  pg/ UR                  R                  5       QmUb  [        U[        5      (       a  [        U5      S:X  d   eUu  p[        U[        5      (       d   e[        U	[        5      (       d   e[        S U	 5       5      (       d   eUR                  n
X:X  d	   U
U45       eU	 Vs/ s H  oT;  d  M
  UPM     n	nTU	-  m/ UR                  5       Qm[        R                  R                  T [        R                  5      (       d  TR!                  UR#                  5       5        UUU 4S jnXg-   n[%        ['        T 5      5      (       + =(       d    [(        R*                  (       + nU" UUUU5      u  nnnU" X}XN5      u  nnn[,        R.                  " UUSS9u  u  nnn[1        UU" U5      U" U5      /UUU5      nUU4U4$ s  snf )a  
This is a main place where we do loop transformations in a
backend-agnostic way.

Here we:
    1) Remove any 1 dimensions
    2) Fuse contiguous dimensions together
    3) Reorder dimensions based on stride orders

Optional argument extra_indexing_constraints can be used to append additional
indexing expressions to existing ones derived from buffer's body. This can be useful
to fuse scheduler nodes with compatible ranges, e.g. (s0*s1*...,) and (s0, s1, s2, ...)
on CPU by preventing indexing simplifications and obtaining index/reduce ranges for
the scheduler node compatible with other nodes.
Optional argument recompute_sizes_body_func can be used to recompute sizes and body
on the default body. This can be useful to append additional loop transformations.
r   c              3  B   #    U  H  n[        U[        5      v   M     g 7fr   )r{   r   )r  fs     rx   r  6ComputedBuffer.simplify_and_reorder.<locals>.<genexpr>t  s     H4Gqz!T**4Gs   c           	        > TR                  XUT
5      u  p$nU" U 5      n U(       aD  [        R                  R                  R	                  U U[        T	X5      5      u  p&n[        XV5      nOUnX(U4$ r   )_apply_loop_reorderingrV   r   r   _simplify_loopsr6   r   )x_varssupport_varsr  simplify_loopsreindex0r   r   _pruner   index_formulasmemory_addrsr;  s            rx   simplify_and_reorderAComputedBuffer.simplify_and_reorder.<locals>.simplify_and_reorder  sy    (,(C(Ce\)%EX f%F*+''*:*:*J*J,^VK+'
 *(="8++rw   pr`   )r  indexing_exprsr   r{   r}   r   r~   r|   r	  r  get_write_exprsrV   r   r  r4   PREFER_STORE_LOOP_ORDERextendget_read_exprsrN   r   r2   loop_ordering_after_fusionr3   index_vars_no_squeezer<   )r;  extra_indexing_constraintsrecompute_sizes_body_funcr  r  r  r  r  extra_indexing_rangesextra_indexing_exprexpected_var_rangesr  r  r  should_merge_loopsiter_rangesiter_reindexr   reduce_rangesreduce_reindex	iter_varsr  r  r  s   `                     @@rx   r  #ComputedBuffer.simplify_and_reorderF  sV   4 '')		
%Z%Z %
 *)4k1J	))
 94..5578%15u==23q89 :T6!3T::::148888H4GHHHHH"&//&? #%B ? /#.a>2I.   # 11N0--/0ww""4)O)OPP 3 3 56	,$ "/t,--VV5V5V1V 	 (<	(
$\1 ,@{,
(~q
 0</Q/Q0
, K*
 )$n[&AB
 ]+T11w#s   %	G?2G?c           
     n   SSK Jn  Uc  / n U Vs/ s H-  n[        R                  R                  R                  X`U5      PM/     nn[        U5      [        U5      :X  a  [        US   5      [        U 5      :X  d   e[        [        U" XrU5      5      5      nU V	s/ s H  oU	   PM	     nn	U[#        U5      [%        U5      4$ s  snf ! [         a^    [        R                  (       a)  [        R                  S[        [        X5      5      U5        [        [!        [        U5      5      5      n Nf = fs  sn	f )zE
Shuffle the order of loops around to hopefully improve performance.
r1   r  r   z%Did not simplify complex index:
%s
%s)r  r  rV   r   r   r  r   r|   r_  	Exceptionr2   r  r  warningr~   r   r   r   r   )
r  r  r  r  priority_idxr  rl  r&  r   r   s
             rx   r  %ComputedBuffer._apply_loop_reordering  s#    	/L	, )(D   --dM(   w<3|#44WQZCM :   /',"OPQE $))5aq5)l5)?5+AAA#  	,||=Z/0 
 s5z*+E	, *s*   C 4CAC D2C A%D/.D/c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  !ComputedBuffer.get_reduction_size      yy++--rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r  !ComputedBuffer.get_reduction_type  r  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  ComputedBuffer.is_no_op  r>  rw   c                    gNTro   rB  s    rx   rJ  ComputedBuffer.should_allocate  r]  rw   c                8    U R                   R                  U5      $ )r  r  r  r  s     rx   r  !ComputedBuffer.constant_to_device  s    yy++F33rw   ro   r%  r@  r?  r  r>  r~  r,  )r   zCallable[..., None])r   zOptional[list[int]]r  )r   zetuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody, tuple[list[sympy.Expr], list[sympy.Expr]]]NN)r  *Optional[tuple[dict[Any, Any], list[Any]]]r  Optional[Callable[..., Any]]r   z:tuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody]r   rD  r   rF  )rq   rr   rs   rt   ru   r|  r  rN  rO  r  r  r  r  r   r  rF   r  r  rJ  r  r  r  r  rJ  r  rv   r  r  s   @rx   r  r    s    
K	%%*
2%G%N% J
J JD RVBFq2$Nq2 $@q2 
D	q2f  !B !BF..,4 4rw   r  c                     ^  \ rS rSrSr        SU 4S jjrSS jrS rSS jrSS jr	SS jr
  S   SS	 jjrS
rU =r$ )TemplateBufferi  zh
Represents a Triton (in the future other type) of template operator
that we can fuse an epilogue onto.
c                   > [         TU ]  S US9  [        R                  U5      U l        X0l        [        R                  R                  U 5      U l	        [        R                  R                  U 5        g rb  )r  rC  InputsKernelunwrap_storageinputsmake_kernel_renderrV   r   register_bufferr   register_operation)r;  r   r  r  r  s       rx   rC  TemplateBuffer.__init__  sW     	d62"11&9"4GG++D1		""4(rw   c                     U R                  SS9$ )NT	normalize)r:   rB  s    rx   r  TemplateBuffer.get_read_writes   s    ''$'77rw   c           
       ^^^ U R                  5       mU R                  5       R                  5       mUU4S jn[        R                  " X R                  5       SUS9nU R                   Hf  mTR                  R                  5       mUU4S jnU=R                  [        R                  " UTR                  5       SSS9R                  -  sl        Mh     U$ )Nc                b   > [        U5      S:X  d   e[        R                  " TT" U 5      S5      $ )Nr   fake)r   rT   r  )r   rb  r  r   s     rx   dummy1TemplateBuffer.extract_read_writes.<locals>.dummy  s,    v;!###99T75>6::rw   ro   r  c                ~   > [        U5      S:X  d   e[        R                  " TR                  5       T" U 5      5        g r  )r   rT   r  r  )r   rb  r  inps     rx   r  r    s-    6{a'''8rw   T)	r  r   r  r3   r:   r   r  r   r  )r;  r  r  depsr  r  r   s       @@@rx   r:   "TemplateBuffer.extract_read_writes  s    }}//#002	; //==?B)
 ;;Cjj--/G9 JJ,::s||~rTeJ  rw   c                6    [         R                  R                  $ r   )r   r  r  rB  s    rx   r  !TemplateBuffer.get_reduction_size  s    ww{{rw   c                    g r   ro   rB  s    rx   r  !TemplateBuffer.get_reduction_type  r]  rw   c                    gr  ro   rB  s    rx   rJ  TemplateBuffer.should_allocate"  r]  rw   c                *    U R                  5       S4S 4$ r  r  )r;  r  r  s      rx   r  #TemplateBuffer.simplify_and_reorder%  s$      
 	
rw   )r  r  r   )r   r  r  Sequence[IRNode]r  r  r   r   r>  rD  r%  r   r  )r  r  r  r  )rq   rr   rs   rt   r  rC  r  r:   r  r  rJ  r  rv   r  r  s   @rx   r  r    ss    

)
) !
) /	
)
 

)82
 RVBF
$N
 $@
 
rw   r  c                  \   ^  \ rS rSr  S     SU 4S jjjrS	S jrS
S jrSS jrSrU =r	$ )TritonTemplateBufferi3  c                2  > [         T
U ]  XU5        X@l        U /U l        Ub  [        R
                  R                  R                  [        R
                  R                  R                  4n[        R                  R                  R                  nXv;   d   SU SU 35       eU R                  S   R                  5       nU =R                  U V	s/ s H  n	[        [!        US9X5      PM     sn	-  sl        U(       a  UU l        g[#        5       U l        gs  sn	f )a  
NOTE:[TritonTemplates with multiple outputs]
We want the ability for TritonTemplates to output multiple tensors. Triton
kernels have no notion of outputs and this is done by creating tensors that
are then mutated by the kernel. Currenlty our STORE_OUTPUT codegen doesn't
support creating multinode outputs for triton templates.
We work around this by creating an extra input buffer during the lowering
and we mark them as mutated inputs.
Nz$Mutated inputs are only allowed for z	 but got r   r  )r  rC  mutated_inputsoutputsr   rT   higher_orderflex_attentionflex_attention_backwardrV   r   current_noder]  r  r   MutationOutputr  r,   allowed_prologue_inps)r;  r   r  r  r  r  allowed_setr
  r   r  r  s             rx   rC  TritonTemplateBuffer.__init__4  s   " 	);<,&*V% 		&&55		&&>>K 77//66L. 6{m9\N[. [[^..0FLL))C z8#D) L &;! 	"@J 	"s   Dc                    U R                   $ r   )r  rB  s    rx   rq   TritonTemplateBuffer.get_outputs\  r  rw   c                    U R                   $ r   )r  rB  s    rx   get_allowed_prologue_inps.TritonTemplateBuffer.get_allowed_prologue_inps_  s    )))rw   c                &    SU R                    S3nU$ )NzTritonTemplateBuffer(layout=r   r  )r;  r   s     rx   r  TritonTemplateBuffer.__str__b  s    ,T[[M;
rw   )r  r  r  r  )r  zOptional[Iterable[IRNode]]r  zOptional[OrderedSet[str]]r   r   r|  r  r2  )
rq   rr   rs   rt   rC  rq  r  r  rv   r  r  s   @rx   r  r  3  sH     6:;?&

 3&
  9&
 
&
 &
P* rw   r  c                     ^  \ rS rSrSr          SU 4S jjrSS jrSS jrS rSS jr	SS jr
SS	 jrSS
 jrSrU =r$ )ChoiceCallerij  a  
Represents a possible choice used in autotune_process.py.
During autotuning, self.benchmark() is first called to get benchmark result,
and if this choice is selected, self.output_node() is called to get the output_node.

Children classes: TritonTemplateCaller, CUDATemplateCaller.
c                R   > [         TU ]  5         Xl        X0l        X l        X@l        g r   )r  rC  r   r   ri   description)r;  r   ri   r   r  r  s        rx   rC  ChoiceCaller.__init__s  s(     		& 'rw   c               T    U R                  5       n[        R                  " X2SU05      $ )Nr   )to_callablerA   	benchmark)r;  r   r   algos       rx   r  ChoiceCaller.benchmark  s'    !$$T%>>rw   c                    [         er   rU  rB  s    rx   	call_nameChoiceCaller.call_name  rW  rw   c                    [         er   rU  rB  s    rx   r  ChoiceCaller.to_callable  rW  rw   c                    [         er   rU  rB  s    rx   hash_keyChoiceCaller.hash_key  rW  rw   c                    [         er   rU  rB  s    rx   output_nodeChoiceCaller.output_node  rW  rw   c                    0 $ )zRInformation returned here is logged to the autotune log file when that is enabled.ro   rB  s    rx   	info_dictChoiceCaller.info_dict  s    	rw   c                    g)Nunsupported_choicero   rB  s    rx   autoheuristic_idChoiceCaller.autoheuristic_id  s    #rw   )r  ri   r   r   )
r   r   ri   r}  r   r  r  r   r   r   )r   r  r2  )r   rb   )r   z<dict[str, Union[PrimitiveInfoType, list[PrimitiveInfoType]]])rq   rr   rs   rt   r  rC  r  r!  r  r&  r)  r,  r0  rv   r  r  s   @rx   r  r  j  se    '' "' 	'
 ' 
'?""""$ $rw   r  c                      \ rS rSrSS jrSrg)TritonTemplateCallerBasei  c                    [         er   rU  rB  s    rx   get_make_kernel_render/TritonTemplateCallerBase.get_make_kernel_render  rW  rw   ro   N)r   r   )rq   rr   rs   rt   r5  rv   ro   rw   rx   r3  r3    s    "rw   r3  c                     ^  \ rS rSrSr            S
U 4S jjr\SS j5       r\SS j5       r\	R                  SS j5       rSS jrSS jrS	rU =r$ )MultiTemplateBufferi  a3  
Represents a Buffer with multiple backing implementation choices.

Choices can be TritonTemplates or ExternKernels. During scheduling if there is a potential
epilogue we will benchmark each of the choices with the epilogue to determine an implementation.
Otherwise, the fastest base choice will be chosen.
c                z   > [         TU ]  UUS US9  X0l        S U l        X l        [        S U 5       5      U l        g )N)r   r  r  r  c              3     #    U  H]  n[        U[        5      =(       dA    [        U[        R                  R                  R
                  5      =(       a    UR                  v   M_     g 7fr   )r{   r3  r   	_inductorselect_algorithmExternKernelCallerhas_out_variant)r  choices     rx   r  /MultiTemplateBuffer.__init__.<locals>.<genexpr>  sU      %
 - v78 65??#C#C#V#VW +**
 -s   A%A')r  rC  _choice_timings_fn_choice_timingsoriginal_inputsr	  _output_plannable)r;  r   r  choice_timingsunfiltered_choicesr  r  s         rx   rC  MultiTemplateBuffer.__init__  sU     	#"7	 	 	
 #1DH%!$ %
 -%
 "
rw   c                    U R                   $ )zN
Are all possible choices TritonTemplates or Extern Kernels with out variants
)rD  rB  s    rx   output_plannable$MultiTemplateBuffer.output_plannable  s    
 %%%rw   c                ^    U R                   c  U R                  5       U l         U R                   $ r   )rB  rA  rB  s    rx   rE  "MultiTemplateBuffer.choice_timings  s+    '#'#:#:#<D ###rw   c              #    #    [        U[        R                  R                  R                  5      (       d   eU R
                  UR
                  :X  d   eU R                  nUR                  5       U l         S v   X l        g ! X l        f = f7fr   )r{   r   r;  r<  TritonTemplateCallerr   r  r5  )r;  callerrenders      rx   swap_as_triton_caller)MultiTemplateBuffer.swap_as_triton_caller  sn     &%//"B"B"W"WXXXX{{fmm+++(("("?"?"A	-&,#f#s   A3B6B :BB		Bc                @   [        U[        R                  R                  R                  5      (       d   eU R                  5       UR                  R                  :X  d   eU R                  5       UR                  R                  :X  d   eUR                  5       U l        g r   )r{   r   r;  r<  rN  r   r   r   r  r   r5  r  )r;  rO  s     rx   finalize_as_triton_caller-MultiTemplateBuffer.finalize_as_triton_caller  ss    &%//"B"B"W"WXXXX}}&--"4"4444 FMM$8$8888"("?"?"Arw   c                r    [        U R                  U R                  R                  S9nXR                  U   4$ )Nr|  )r  rE  ry  )r;  
min_choices     rx   get_min_choice"MultiTemplateBuffer.get_min_choice  s4    ,,$2E2E2I2IJ
//
;<<rw   )rB  rA  rD  r  rC  )r   r  r  rj   rE  z'Callable[[], dict[ChoiceCaller, float]]rF  zlist[ChoiceCaller]r  r  r   r   r   )r   zdict[ChoiceCaller, float])rO  r3  )rO  r3  r   r   )r   ztuple[ChoiceCaller, float])rq   rr   rs   rt   r  rC  rM  rI  rE  rK  rL  rQ  rT  rX  rv   r  r  s   @rx   r8  r8    s    

 
 @	

 /
  /
 

4 & & $ $
 	- 	-B= =rw   r8  c                  >   ^  \ rS rSr      SU 4S jjrS rSrU =r$ )CUDATemplateBufferi  c                >   > [         TU ]  XU5        X@l        XPl        g r   )r  rC  workspace_sizetemplate)r;  r   r  r  r]  r^  r  s         rx   rC  CUDATemplateBuffer.__init__  s      	);<, rw   c                8    U R                   b  U R                   $ S$ r  )r]  rB  s    rx   rz  %CUDATemplateBuffer.get_workspace_size  s    &*&9&9&Et""L1Lrw   )r^  r]  )r]  r   r^  rX   r   r   )rq   rr   rs   rt   rC  rz  rv   r  r  s   @rx   r[  r[    s2    !
 ! ! 
!M Mrw   r[  c                  <   ^  \ rS rSrSU 4S jjrSU 4S jjrSrU =r$ )CppTemplateBufferi  c                L   > [         TU ]  XU5        X@l        XPl        S U l        g r   )r  rC  r^  r?  r  )r;  r   r  r  r^  r?  r  s         rx   rC  CppTemplateBuffer.__init__  s$    );< /3rw   c                6  > [        U R                  [        5      (       al  [        U R                  [        5      (       d   eU R                  S   n[        U[
        5      (       d   eUR                  n[        U[        5      (       d   eU$ [        TU ]!  5       $ r  )	r{   r   MultiOutputLayoutr  r   r  r  r  r   )r;  first_outputr   r  s      rx   r   CppTemplateBuffer.get_layout  s}    dkk#455dllH5555<<?LlF3333!((Fff----M7%''rw   )r?  r  r^  r  r  )rq   rr   rs   rt   rC  r   rv   r  r  s   @rx   rc  rc    s    4	( 	(rw   rc  c                  l    \ rS rSr% S\S'   SS jrSS jr\SS j5       r\	S 5       r
SS jrSS	 jrS
rg)r  i  r}  r  c                  ^ [         [        R                     " 5       n[        R                  mU R                   Hq  n[        U[        5      (       a  UR                  U4S jU 5       5        M5  [        U[        5      (       a  ML  UR                  T" UR                  5       5      5        Ms     [         [        R                     " U4S jU R                  5        5       5      n[        R                  " UU[        5       S9$ )Nc              3  P   >#    U  H  nT" UR                  5       5      v   M     g 7fr   r9  )r  r   r  s     rx   r  /InputsKernel.get_read_writes.<locals>.<genexpr>  s     BEqWQZZ\22E   #&c              3  P   >#    U  H  nT" UR                  5       5      v   M     g 7fr   r9  )r  r  r  s     rx   r  rm    s#      .
/AGCLLN##/Arn  )r  writesindex_exprs)r,   r3   r7   r  r  r{   r|   updater   r  r  rq  
ReadWrites)r;  r  inputrp  r  s       @rx   r  InputsKernel.get_read_writes  s    <++,.&&[[E%&&BEBBE#899		'%.."234 ! L,,- .
/3/?/?/A.
 
 &&"
 	
rw   c                6    U R                  5       R                  $ r   r  rB  s    rx   rN  InputsKernel.get_reads%  r  rw   c                   [        U[        5      (       a  UR                  n[        U[        5      (       a  UR                  n[        U[        5      (       a*  [        U[
        5      (       d  [        R                  U5      n[        U[        5      (       a  U R                  U5      $ [        U[        5      (       a  U$ [        U[        [
        45      (       d   U5       eU$ r   )r{   rb   r  r  r  r$  r  realize_inputunwrap_storage_for_inputTorchBindObjectr  r  r   s     rx   rz  %InputsKernel.unwrap_storage_for_input(  s    a##Aa$$Aa"":a+I+I**1-Aa##
 //22a))H!fo677::7rw   c                    / nU  Hd  n[        U[        5      (       a&  U Vs/ s H  n[        R                  U5      PM     nnO[        R                  U5      nUR	                  U5        Mf     U$ s  snf r   )r{   r|   r  rz  r  )r  
inputs_newr   r   s       rx   r  InputsKernel.unwrap_storage;  sk    
A!T""GHIq!\::1=qI 99!<a   	 Js   A/c                    gr  ro   rB  s    rx   r  InputsKernel.is_externF  r]  rw   c                    gr  ro   rB  s    rx   r  InputsKernel.num_readsI  r  rw   ro   Nr>  r?  )r   rc   r   rc   r   r@  )rq   rr   rs   rt   ru   r  rN  r  rz  rJ  r  r  r  rv   ro   rw   rx   r  r    sD    
,,  $  rw   r  c                  (    \ rS rSrSS jrSS jrSrg)	NopKerneliM  c                    gr  ro   rB  s    rx   r  NopKernel.is_no_opN  r]  rw   c                    [        5       $ r   r+   rB  s    rx   rN  NopKernel.get_readsQ  rv  rw   ro   Nr   r?  )rq   rr   rs   rt   r  rN  rv   ro   rw   rx   r  r  M  s    rw   r  c                  V    \ rS rSrSr\S 5       r\S	S j5       r\S 5       rS
S jr	Sr
g)ConcatKerneliU  zb
There isn't actually a real kernel for concat, we just change the
storage for the upstream data.
c                	   US   R                  5       nUS   R                  5       n[        US   R                  5       5      nS/nXR   /nSUs=::  a  [	        U5      :  d   e   e[        S[	        U5      5       H  nX   R                  5       n	UR                  XR   5        [	        U	5      [	        U5      :X  d   eX   R                  5       U:X  d   eX   R                  5       U:X  d   e[        [	        U5      5       HE  n
X:X  a  XZ   X   -   XZ'   M  [        R                  R                  R                  XZ   X   5      XZ'   MG     UR                  XR   5        M     [        R                  U5      n[        R                  (       a#  [        R!                  XUS   R"                  5      n[        [	        U5      5       H|  nX   n[%        U5      (       d  M  UR'                  5       n[)        U[*        5      (       d  M@  [        R-                  UR.                  UR0                  5      (       d  Mq  [3        U5      n  O   [5        S U 5       5      n[        R                  R6                  R8                  S   n[)        U[        5      (       d   eUSL a"  [5        S U 5       5      (       a  [3        U5      n[;        S [+        UUUUS9/ S9n[=        U5      n/ n[        [	        U5      5       GH  nU R?                  X   [@        RC                  UX&U   Xx   SS95      nURD                  R                  U5        [)        X   RF                  [H        5      (       a  X   RF                  RK                  5       nOX   RF                  nURM                  5       (       d  M  [O        X   R                  5       RP                  5      (       d  M  [S        U5      (       a  M  UR                  URU                  5       5        GM     [	        U5      S:  aR  [        R                  RW                  U[X        RZ                  5      (       a  [        R                  R]                  U5        [        R                  R_                  U5      Ul0        U Rc                  URD                  5      Ul"        [        R                  Re                  U5        U$ )	Nr   r1   c              3  8   #    U  H  n[        U5      v   M     g 7fr   )r   r  s     rx   r  &ConcatKernel.create.<locals>.<genexpr>  s     -WPV1.CA.F.FPVrL  Fc              3    #    U  Hv  nS UR                   ;   =(       a[    UR                   S    R                  [        R                  S9=(       d*    UR                   S    R                  [        R                  S9v   Mx     g7f)r  r  N)rs  r  r   r  r  )r  args     rx   r  r    sr      <
 $ SXX --E<O<O-P W88E?00u?U?U0V
 $s   A>B )r   r   r   r   r   r   r  )r  )3r   r   r|   r   r   r   r  rV   r   r   r  r   r   r2   r  r  r  r   r   r   r{   r"  r\  r   r   r"   r  r
  r   r  r  r  r  r  r  r  r  r  is_input_bufferrN   r   rM   r  r  r4   FOREACHregister_operation_listr  r   r  r  )r  r  r  r   r   rP  offsets_startoffsets_endr   
input_sizer  output_strider   r   any_input_is_storage_and_layoutfx_node_argsconcat_kernelkernelop_namesinput_bufferinput_unwrappeds                        rx   r  ConcatKernel.create[  s   %%'q	##%q	**,-}oC'#h-'''''q#f+&A++-J  /z?c(m3339&&(E1119'')V3333x=)8"*+
"=HK"#''"2"2"?"? Z]#HK	 * x}- ' '99(C''"//M
 s6{#A	A$Q''K 88fmmTT$B8$LM $ +.-WPV-W*W'ww++003,----*e3 <
 $<
 9
 9
 ;8DM$$	 	
 M*s6{#A++	  Cq!1;> ! L   ''5&)..(33"().."<"<">"()..  //1169//16677"<00 ? ? AB' $* x=1!4!4V^=S=S!T!TGG++H5WW44]C"11-2F2FG	""=1rw   Nc                   [        U[        5      (       a  U R                  UR                  U5      $ [        UR                  [        5      (       a  [        UR                  R
                  [        5      (       a  UR                  R                  (       d  gUc  g[        UR                  5       5      [        UR                  5       5      :X  d  g[        S [        UR                  5       UR                  5       5       5       5      $ [        UR                  R
                  [        5      =(       a    [        UR                  [        5      (       + $ )NFTc              3  x   #    U  H0  u  p[         R                  R                  R                  X5      v   M2     g 7fr   r  r  s      rx   r  =ConcatKernel.can_realize_into_without_copy.<locals>.<genexpr>  s1      EFB   88@@Er  )r{   rb   can_realize_into_without_copyr  r8  r   r"  rI  r   r  r	  r   r   ExternKernelAlloc)r  r  r  s      rx   r  *ConcatKernel.can_realize_into_without_copy  s    c9%%44SXXsCCchh 344sxx<<xx00 { s~~'(C0@,AA !#.."2CNN4DE  
 #((//>: 
:HH'D
 @
 	
rw   c                   [        U[        5      (       d&  [        U5      (       a  [        U5      u  p4[        X4S9n[        U[        5      (       d   U5       e[        U[        5      (       a  U R                  UR                  U5      $ [        U[        5      (       ai  UR                  5         [        UR                  S5      (       d   eU R                  X5      (       a&  [        U5      UR                  l        UR                  $ [        R                  UR                  5       UR!                  5       UR#                  5       [%        UR'                  5       UR'                  5       5       VVs/ s H.  u  pV[(        R*                  R,                  R/                  XV5      PM0     snnS9nU R                  Xr5      $ s  snnf )Nr  r   r  )r{   r$  r   r!  rb   r  r  r  r  rZ  r  r  r   r  r  r   r   r  r   r   rV   r   r   r  )r  r  r  r'  r   r'  r(  pws           rx   r  ConcatKernel.realize_into  sV   
 #//$S))"7"<%7B#//44/c9%%##CHHc22c:&&KKM388X....00::"1#"6xx>>#--/__&  ??DA   --a3?	  
 ((s   .5F:c                    gr  ro   rB  s    rx   rJ  ConcatKernel.should_allocate  r]  rw   ro   r   r   )rq   rr   rs   rt   r  r  r  r  r  rJ  rv   ro   rw   rx   r  r  U  sL    
 ^ ^@ 
 
< ) )@rw   r  c                    ^  \ rS rSr% SrS\S'   \R                  " \S9r	S\S'   Sr
S	\S
'   SrS\S'   SrS\S'   \R                  " \S9rS\S'   SrS\S'   SrS\S'   SrS\S'   \R                  " \S9rS\S'   \R                  " \S9rS\S'          S; S<U 4S jjjrS=S jrS>S jrS rS rS<S jrS  rS?S@S! jjrSAS" jrS# r\S$ 5       r\   SBS% j5       r!\ S& 5       r"\ S' 5       r#\ S( 5       r$\    SC   SDS) jj5       r%\ SES* j5       r&\ SES+ j5       r'\ S, 5       r(\ S- 5       r)\ S. 5       r*S<S/ jr+S0 r,S?SFS1 jjr-S2 r.S3 r/SES4 jr0S<S5 jr1S6 r2S7 r3S>S8 jr4SGS9 jr5\5r6S:r7U =r8$ )Hr  i   ro   ztuple[Any, ...]constant_argsr  zdict[str, Any]r   NzOptional[ReinterpretView]output_viewr&  python_kernel_namecpp_kernel_namezIterable[str]ordered_kwargs_for_cpp_kernelzFOptional[Union[torch._ops.OpOverload, torch._ops.HigherOrderOperator]]op_overloadzOptional[list[dict[str, Any]]]arg_propertiesz#Optional[dict[str, dict[str, Any]]]kwarg_propertiesz"dict[sympy.Symbol, pytree.KeyPath]unbacked_bindingszlist[MutationOutput]mutation_outputsc                2  > [         TU ]  UUUS9  X@l        U(       a  UO0 U l        X`l        Xl        U R                  U5        U R                  U5        Xl        U R                  5         0 U l
        / U l        [        R                  R                  U l        g Nr  )r  rC  r  r   r  r  set_cpp_kernel_nameset_python_kernel_namer  collect_arg_kwarg_propertiesr  r  rV   r   r
  fx_node)r;  r   r   r  r  r   r  r  r  r  r  r  s              rx   rC  ExternKernel.__init__  s     	 	 	

 + &fB&&  1##$67-J*))+!# "ww++rw   c                     U /U R                   Q$ r   r  rB  s    rx   rq  ExternKernel.get_outputs4  s    -t,,--rw   c                    [        5       $ r   r+   rB  s    rx   rt  %ExternKernel.get_unbacked_symbol_defs7  rv  rw   c                   [        U R                  [        R                  R                  5      (       af  U R                  R
                  R                   Vs/ s H:  nUR                  (       a  M  UR                  UR                  UR                  S.PM<     snO.[        [        U R                  5      5       Vs/ s H  n0 PM     snU l        [        U R                  [        R                  R                  5      (       aS  U R                  R
                  R                   Vs0 s H'  nUR                  UR                  UR                  S._M)     snO0 U l        [        U R                  [        R                  R                  5      (       a  U R                   (       dR  U R                  R
                  R                   Vs/ s H!  oR                  (       d  M  UR                  PM#     snU l        U R                  R
                  R                   Vs/ s H  oR                  (       d  M  UPM     snU l        g g s  snf s  snf s  snf s  snf s  snf )N)r   r   r  )r   r  )r{   r  r   _ops
OpOverload_schema	arguments
kwarg_onlyr   	real_typer  r   r   r  r  allarg_propertiesr  schema_kwargs)r;  r   r   s      rx   r  )ExternKernel.collect_arg_kwarg_properties:  s    $**EJJ,A,ABB ))11;; <A||FFKK%&__
 < $C$4565"56 	$ $**EJJ,A,ABB ))11;;;A qOO;
  	 d&&

(=(=>>55$($4$4$<$<$F$F6$Fq,,FAFF$F62  ++33=="=a="D ?- 76"s0   H;/(H;:I #.II
I
I+Ic                    [        U R                  [        5      (       a!  U R                  5         U R	                  5         g g r   )r{   r   r   apply_constraintr  rB  s    rx   r  ExternKernel.decide_layout]  s0    dkk>22!!#  3rw   c                P    [        X5      u  p#U(       a  UR                  U5        g g r   )rK   r  )r;  wrapper
origin_str_detailed_origin_strs       rx   codegen_commentExternKernel.codegen_commentb  s%    +>t+M(
j) rw   c                    [         er   rU  r;  r  s     rx   codegenExternKernel.codegeng  rW  rw   c                   Xl         [        R                  R                  (       a3  [	        U R
                  [        R                  R                  5      (       d  g U R
                  nU R                   c  UR                  S:X  aV  UR                  S:X  a  UR                  R                  S5      S   OUR                  R                  SS5      nSU S3U l         g UR                  R                  U l         g g )Natenr  .r   r   z
at::_ops::z::call)r  rV   r   cpp_wrapperr{   r  r   r  r  	namespace_overloadnamerq   r  replacer  r   )r;  r  r  opnames       rx   r   ExternKernel.set_cpp_kernel_namej  s    .ww""*ejj33+
 +
 !!'6) ++y8 OO))#.q100c: 
 *4F86'B$'-~~':':$ (rw   c                   Xl         Ub  g U R                  nUc  g [        U[        R                  R
                  5      (       a  SUR                   3U l         g UR                  R                  SS5       SUR                   3U l         g )Nztorch.ops.higher_order.z._ops.z.ops.r  )	r  r  r{   r   r  HigherOrderOperatorrq   rr   r  )r;  r  r  s      rx   r  #ExternKernel.set_python_kernel_name  s    "4)!!>

 > >??(??P&QD# $$,,Xw?@&//ARS #rw   c                :   U R                  5       =n(       a  UR                  O[        R                  R                  n[        R                  R
                  (       a4  [        R                  R                  R                  U R                  U5      $ U R                  $ r   )
r   r   rV   r   device_typer  r  get_c_shim_func_namer  r  )r;  dr   s      rx   get_kernel_nameExternKernel.get_kernel_name  sn    !%!22A29L9L ww"" GG  55d6J6JFS	
 ((	
rw   c           	         [         R                  U R                  5       U R                  5       U R	                  5       U R                  5       U R                  5       U R                  5       S9nUR                  5         U$ )N)r   r   r  r  r4  r2  )	r  r  r   r   r  r   rW  rS  r  )r   r  s     rx   
copy_inputExternKernel.copy_input  sa    <<>++-]]_::<))+oo'  
 	

	rw   c                  ^^ X#S.n[         R                  " U5      u  nm/ m/ n/ nU H  nTR                  [        U[        5      =(       a    [        U[
        5      (       + 5        TS   (       a  UR                  U5        M[  [        U[        R                  5      (       a2  [        R                  R                  R                  R                  US S9nUR                  U5        M     UU4S jn	U V
s/ s H  oR                  U
5      PM     nn
U H  n
[        U
5      (       d  M  [        U
SS9  M!     / nU GH  n
[        U
[         5      (       dh  U
R#                  5       [        R                  R$                  ;   a<  UR                  [        R                  R$                  U
R#                  5          5        M  [        U
[         5      (       dh  U
R#                  5       [        R                  R&                  ;   a<  UR                  [        R                  R&                  U
R#                  5          5        M  [        U
[(        5      (       a"  UR                  U
R+                  5       5        GM5  [        U
[,        R.                  R0                  R
                  5      (       ar  U
R2                  R4                  nU
R2                  R6                  S:X  a  Uc   eUR                  [,        R8                  R:                  U   R=                  5       5        GM  UR                  [?        U
SS95        GM     U	" X5      u  pU" U0 UD6nS n[        R@                  R                  =n(       aO  [C        U[        RD                  U5        [G        UU[        RD                  RH                  RK                  S	5      5      n[        U[L        [N        45      (       d  U/OUnU H  n[        U[,        RP                  5      (       d  M$  URR                  (       d  M7  S
n[        R                  RD                  RH                  RK                  SS 5      =n(       a  U SU 3nU[        R                  l*        M     UUUU	U4$ s  sn
f )N)r   r   r  )r  c                6  > / n[        U 5      n[        U5      nT H@  nU(       a  UR                  [        U5      5        M&  UR                  [        U5      5        MB     [        R                  " UT5      nUR                  S/ 5      UR                  S0 5      4$ )Nr   r   )iterr  nextpytreetree_unflattenry  )	new_tensor_argsnew_non_tensor_argsr  
it_tensorsit_non_tensors	is_tensorr  	args_specis_arg_tensors	          rx   unflatten_args3ExternKernel.process_kernel.<locals>.unflatten_args  s    Fo.J!"56N*	MM$z"23MM$~"67	 +
 %%fi8A55$aeeHb&999rw   Tr  r   )r   r  zEsparsity not handled. Please file issue for sparse inference weights.stack_tracez Found from : 
 )+r  tree_flattenr  r{   rc   GeneratorStater   r   rV   r   r   r   create_symintnodery  r   r!  r  r  	constantstorchbind_constantsr{  get_real_objr   r;  irr   r   r   r   default_generatorsclone_stater   	fake_moder'   r
  r%   rs  ry  r|   r}   Tensor	is_sparsedisable_cudagraphs_reason)r  r  r   r   binded_args	args_flattensor_argsnon_tensor_argsr  r  r   example_argsdevice_indexnew_args
new_kwargsexample_outputr  r   example_out_lir   msgr 	  r  r  s                         @@rx   process_kernelExternKernel.process_kernel  sx     $6%22;?	9%'C  3'O
30O,O R ""3'c5::..''**44FFsQUFVC&&s+ 
	: 6AA[((+[A A$Q''%a5   	 A a**qzz|qww?P?P/P##AGG$5$5ajjl$CDq(++JJLAGG$?$??##AGG$?$?

$MNA//##ANN$45Au11@@AA xx~~xx}}.<3KKK##JJ11,?KKM ##$5aT$JK' *  .lL8Z8JN---9-Iq~~~F 9>1>>+>+>+B+B5+I! ntUm<<  	
  A!U\\**q{{{]"#''"6"6";";"?"?t"TT;T E!2;-@C471   
 	
{ Bs   4Q2c                   [        U[        5      (       d   e[        U[        5      (       a  U$ UR                  5       n[        R
                  R                  UR                  5       5      nUc   eUR                  5       nUb  SUR                  ;   a  [        UR                  [        5      (       a  UR                  S   R                  [        R                  S9(       d/  UR                  S   R                  [        R                  S9(       a)  UR!                  [#        UR%                  5       5      5        OUR'                  5         [(        R*                  " UR%                  5       SS9u  pVUS   nUR-                  5       " U5      n[        R
                  R.                  R1                  X5      n[        R
                  R.                  R3                  X5      n	[        R
                  R.                  R5                  X5      n
[7        Xy5      U
-   nX:w  a  [8        R;                  SU	U
U5        [<        e[        UR>                  [A        URC                  5       URE                  5       UR%                  5       U	U
S9S9$ )	z
In order to pass this to an extern kernel we need a
ReinterpretView not a View.  This allows us to avoid some
unneeded copies.
r  r  r  r`   r   z@convert_to_reinterpret_view failed: stride=%s offset=%s index=%sr  r  )#r{   r  r$  r  rV   r   r  r  rW  rs  r   r   r  r   r  r  r  r"   r   r  r3   r  r  r   r  stride_vars
offset_varrO   r  r  rt  r  r"  r  r   )r  r   x_unwrap_viewr  x_unwrap_view_fx_node
index_argsr  r  r   r&  r#  expecteds               rx   convert_to_reinterpret_view(ExternKernel.convert_to_reinterpret_view  s    !X&&&&a))H gg  !7!7!9: # 3 3 5 "-.333=//@@%**51??"'"5"5 @  )--e4BB"'"8"8 C 
 77.}/E/E/GH '')!-!@!@JJL"

  ]
 ,  55eH''""..uA!!,,U?Z1F:IIR	 &%,,.kkmZZ\	
 		
rw   c                (   Uc
  [        5       $ [        U[        R                  [        R                  R
                  R                  [        45      (       a	  [        US9$ [        U[        5      (       aY  [        R                  R                  [        R                  " UR                  UR!                  5       UR#                  5       S95      $ [        U[$        5      (       a  U$ [        U[&        5      (       a  U R)                  UR*                  5      $ [        U[,        5      (       a1  [-        U R)                  UR*                  5      UR/                  5       S9$ [        U[0        5      (       a@  UR3                  5         [5        UR7                  5       5      (       a   U R9                  U5      $ [        U[<        5      (       a  UR3                  5         U$ [        U[>        [        45      (       a  U$ U RA                  U5      $ ! [:         a     Naf = f)N)rl  )r   r   r  )!re  r{   r   r   r   r   r   r   r   r(  rV   r   add_tensor_constantr   r%  r   r   r   r  rb   ry  r  r$  r   r  r  r   r  r"	  rt  r  NonTensorObjr  r|  s     rx   ry  ExternKernel.realize_inputY  s   9'))a%**ekk&9&9&A&A3GHH(a00a""77..QWWAKKM!,,.Q  a((Ha##$$QVV,,a))"&&qvv.q||~  a""IIK$Q]]_55::1== a$$IIKHa,(=>??H~~a   + s   H 
HHc                    [        U5      (       a@  [        UR                  5       5      S:X  a  U$ UR                  5        H  nUS:X  d  M  Us  $    U R                  U5      $ ro  )r   r   r  r  )r  r   r   s      rx   require_stride1ExternKernel.require_stride1z  sR     ##1<<>"a',,.Q;H ) ~~a  rw   c                
   Uc  Uc   eUR                  5       S;   a	  U(       d  U$ [        U5      (       Ga]  [        UR                  5       [        5      (       a  U(       ai  [        USS[        X5      (       aJ  [        [        R                  R                  R                  UR                  5       R                  5      5      OUUS9  U$ [        USSS UUS9  U$ [        UR                  5       [        [        45      (       ay  U(       a$  UR                  5       R                  U5      (       d>  U(       aG  [!        X1R                  5       R                  UR#                  5       5      (       a  Ub  [%        X5      $ U$ [        UR                  5       [&        5      (       a  [        UR                  5       R)                  5       [        5      (       a  [+        S5      e[        UR                  5       R)                  5       [        5      (       a  U(       a2  UR                  5       R)                  5       R                  U5      (       dM  U(       aH  [!        UUR                  5       R)                  5       R                  UR#                  5       5      (       a  U$ [        U[,        5      (       ak  U(       a$  UR                  5       R                  U5      (       d>  U(       a9  [!        X1R                  5       R                  UR#                  5       5      (       a  U$ [        U[.        5      (       a  [        UR0                  [2        5      (       a  [        UR0                  [4        5      (       d  [        UR7                  5       5      (       a}  [        UR7                  5       R0                  [8        5      (       dP   U R;                  UR0                  5      Ul        U(       a  U R=                  XUS9$ U(       a  U R?                  XUS9$  S nUR#                  5       nUb  [        R                  R                  n[C        [E        UR#                  5       5      5       Vs/ s HJ  nURG                  X8   S5      (       d  M  URI                  UR#                  5       U   S	5      (       d  MH  UPML     nnU H.  n	[J        RL                  RN                  RQ                  XSS
5      nM0     U RS                  U5      n[        USSUUUS9  U(       a  [        X5      (       d   e U$ U(       a<  Ub  Uc   e[J        RL                  RN                  RU                  X5      n[%        X5      $ U$ ! [@         a     GNgf = fs  snf )N)r   r1   TF)r  r  r  r  r  zHthe MutationLayoutSHOULDREMOVE's real layout shouldn't be FlexibleLayoutr  r   r   r1   )+r  r   r{   r   r   r!  r  r   rV   r   r   
size_hintsr   r"  r  r  r  r   r+  r  r  r{  rV  rb   r  r  r$  r  r  r"	  require_stride_orderr  rt  r   r   r  r  r   r;  loweringslice_r  r  )
r  r   r   r  r  expanded_dims	orig_sizer   r   r  s
             rx   require_stridesExternKernel.require_strides  sm     M$===;;=F"=H !##!,,..99 *#(-
  B!KK - ! 0 0 ; ;ALLN<Q<Q R "'&3 H *#(-%)&3&3 HALLN[/,JKK1<<>;;EBB!1%||~'<'<ajjl  %0 4AE 
 ALLN,FGGalln88:NKK(b    : : <kJJq||~99;MMeTT%5)LLN668??JJL  H a%%q||~77>>-!<<>#8#8!**, 
 Hq)$$1668,,qvv77%ammo66q}}335FGG88@33 4   #44 5   # .2JJL	$ww''H s1::<011A33M4DaH  11!**,q/1E 1   %OO,,33AAqA %
 NN1!''	
 5a????  (]-FFF((//=A21DDW ' s*   36U *U U-6&U- U-
U*)U*c                "    U R                  XUS9$ )N)r  r  r2	  )r  r   r  r  s       rx   r  "ExternKernel.require_exact_strides!  s    ""- # 
 	
rw   c                "    U R                  XUS9$ )N)r   r  r5	  )r  r   r   r  s       rx   r-	  !ExternKernel.require_stride_order'  s    ""1"OOrw   c                .    U R                  U[        5      $ r   )r-	  r  r|  s     rx   require_channels_last"ExternKernel.require_channels_last+  s    ''+<==rw   c                .    U R                  U[        5      $ r   )r-	  r  r|  s     rx   require_channels_last_3d%ExternKernel.require_channels_last_3d/  s    ''+=>>rw   c                    U R                  U[        [        [        [	        UR                  5       5      5      5      5      5      $ r   )r-	  r|   r_  r   r   r   r|  s     rx   require_contiguousExternKernel.require_contiguous3  s/    ''4s1::<?P9Q0R+STTrw   c                    g r   ro   rB  s    rx   r  ExternKernel.apply_constraint7  r  rw   c                   [        U[        [        45      (       d   e[        U[        5      (       a  [        U5      nU R                  (       d   S5       e[	        U5      n[	        U R                  5      nX4:  aq  [
        R                  SU R                  XC-
  5        [        X45       H?  nU R                  U   S   nUR                  Xb;   a  X&   OU R                  U   S   5        MA     U$ )Nz/ExternKernel.arg_properties should not be emptyzv%s has %d unprovided positional arguments. Will check if they are in the keyword arguments or will use default values.r   r  )
r{   r|   r}   r  r   r  r  r  r   r  )r;  r   r   n_args
n_pos_argsr   arg_names          rx   fill_non_provided_args#ExternKernel.fill_non_provided_args:  s     $u....dE"":D""U$UU"T,,-
 II^  #	 6...q1&9) $,,Q/@ / rw   c                r   [         R                  R                  (       Ga[  / nS nU(       ae  U R                  (       aT  [	        U R
                  5      [	        U5      :X  d   S5       eU R                   Vs0 s H  oDR                  S5      U_M     nn[        U R
                  5       H  u  pVUb.  UR                  X   5      nU(       a  UR                  S5      OS nOb[	        U R                  5      U-   n	U R                  (       a7  U	[	        U R                  5      :  a  U R                  U	   R                  S5      OS nUR                  [         R                  R                  R                  Xh5      5        M     U$ [        [         R                  R                  R                  U R
                  5      $ s  snf )NzDnames passed to codegen_const_args does not match self.constant_argsr   r   )rV   r   r  r  r   r  ry  r   r  r  r  val_to_arg_strrg  )
r;  r  r  name_to_arg_propertiesr  r   r   proptype_r   s
             rx   codegen_const_argsExternKernel.codegen_const_args\  si   77F
 &*",,4--.#e*< Z< 594G4G*4GSGGFOS(4G ' * "$"4"45)5155eh?D04DHHV,$Edkk*Q.C  ..3T=P=P9Q3Q ++C044V<! 
 agg22AA!KL 6 Mqww++::D<N<NOO%*s   4F4c                    [         R                  R                  (       aD  U R                  b7  U R	                  / U R
                  QU R                  QU R                  5      nSnOU R
                  nSn/ n[        U5       H  u  pE[         R                  R                  (       a  U R                  (       a  U[        U R                  5      :  d   S5       eU R                  U   R                  S5      nUR                  [         R                  R                  R                  XV5      5        M  UR                  [         R                  R                  R                  U5      5        M     U(       a  UR                  U R!                  5       5        U$ )NFTz-Invalid access to ExternKernel.arg_propertiesr   )rV   r   r  r  rH	  r  r  r   r   r  r   ry  r  r  rK	  r  rO	  )r;  r  need_codegen_constant_argsr   r   r   rN	  s          rx   codegen_argsExternKernel.codegen_args|  s*   774#3#3#?003$++3 2 23T[[F */&[[F)-&f%DAww""**q3t7J7J3K/K CK ++A.226:AGG00??IJAGG00??BC & &KK//12rw   c                2   X;   a  UR                  U5      $ XR                  ;   a  U R                  R                  U5      $ U R                  (       a9  XR                  ;   a*  U R                  R                  U5      R                  S5      $ [        U S35      e)zGiven an argument name, queries for values in (in order):
1. any provided kwargs for this function.
2. the class self.kwargs member.
3. any available default arguments in self.allarg_properties.r  z not in self.allarg_properties)ry  r   r  r{  )r;  rG	  r   s      rx   get_kwargs_valueExternKernel.get_kwargs_value  s    
 ::h''{{";;??8,,!!h2H2H&H))--h7;;OLLz)GHIIrw   c           	     0   [         R                  R                  (       Ga  U R                  b  [	        U R
                  5      S:X  a  / $ / nU R                   H  nU(       a  US:X  a  M  U R                  U5      n[        U[        R                  5      (       a  UR                  U5        MU  U R                  (       a9  X0R                  ;   a*  U R                  R                  U5      R                  S5      OS nUR                  [         R                  R                  R                  XE5      5        M     U$ U R                   R#                  5        VVs/ s H3  u  pdU S[         R                  R                  R                  U5       3PM5     nnnU$ s  snnf )Nr   r   r   r  )rV   r   r  r  r   r  r  rV	  r{   r   r   r  r  ry  r  rK	  r   r  )r;  skip_outr   rG	  r  rN	  ks          rx   codegen_kwargsExternKernel.codegen_kwargs  sK   77+D4F4F0G10L	F >>E 1))(3a,,MM!$  11hBXBX6X ..228<@@H! 
 MM!''"6"6"E"Ea"OP ?(  !KK--//DA #Qqww++::1=>?/   	s   :Fc           	        [         R                  (       a  [        R                  R                  (       d  [        U R                  5       5      S:X  a  g [        R                  R                  R                  U R                  5       5      n[        R                  R                  R                  U R                  5       5      nUR                  SU R                  5        SU SU S35        g g g )Nr   zassert_size_stride(r  r   )r2   size_assertsrV   r   r  rR   r   r  codegen_shape_tupler  r  r  )r;  r  r   r   s       rx   codegen_size_asserts!ExternKernel.codegen_size_asserts  s    qww':':T]]_-277'';;DMMOLDWW))==doo>OPF%dmmo%6bbJ (;rw   c                N    U R                  5       nU R                  5       nU/ /U4$ )z4
get output sizes and strides, for template_codegen
)r   r  )r;  _size_strides      rx   get_group_strideExternKernel.get_group_stride  s*     //#r{G##rw   c                   [         R                  R                  nU R                  5       nU R	                  5       nU Vs/ s H  oAR                  U5      PM     nn[        [        U5      5       Vs/ s H  n[        SU 35      PM     nn[        [        [        U5      5      UR                  SS9n[        U5       VV	s0 s H  u  pX_M	     n
nn	[        [        U
5      5       Vs/ s H  oZU   PM	     nnU Vs/ s H  oVU   PM	     nnU R                  5       nU" U5      n[         R                  R                  R                  XbU/5      u  pn[        S5      u  nn[        [!        Xo" U Vs/ s H  nU" U5      PM     sn5      5      5      n[#        [$        R&                  " U5      U5      nU[)        U5      4$ s  snf s  snf s  sn	nf s  snf s  snf s  snf )z3
Manually get canonicalization of the output index
r  T)r}  r  c)rV   r   r   r   r  r   r   r   rP   r  r  r   r  r  r;   r~   r   rS   r   r  r}   )r;  r   r  r&  r   r   r  index_orderr   r   r   r   r  r   	new_sizesr   r  r   add_varreplacements                       rx   canonicalizeExternKernel.canonicalize  s   
 77##//#29:'Q%%a(':;@U;LM;La(1QC1;L
MU3w<0g6I6ISWX+4[+AB+Axs#(+AB$)#f+$67$6q$67-23UmU
3##%
#%&WW%5%5%E%Ew&
"	F !%
73z7	3R	1GAJ	3R+STU5<<.<eI&&&+ ;M C73 4Ss#   F=1G G'G;G/Gc                    [         [        R                     " 5       nU R                   H  nU[	        U5      -  nM     U R
                  R                  5        H  nU[	        U5      -  nM     U$ r   )r,   r   r   r  maybe_free_unbacked_symbolsr   r   )r;  r  r  s      rx   r  %ExternKernel.get_unbacked_symbol_uses  sa     u||$&%%C,S11A &;;%%'C,S11A (rw   c           
     ,   [        U SS 5      nSU< 3/nU[        R                  " U 5       Vs/ s H'  nUR                   S[        XR                  5       3PM)     sn-  nUR	                  SU R
                  < 35        U R                  U5      $ s  snf )Nr  zpython_kernel_name=r  r  )r   rH  fieldsr   r  r4  rm  )r;  kernel_namerj  rI  s       rx   r  ExternKernel.__str__  s    d$8$?!+1
 	$++D1
1 zzl!GD**5671
 	
 	|D$4$4#789u%%
s   .B)r  r  r  r  r  r   r  r  r  r  r  r  r  ro   NNNNro   Nr  r|  r~  r   r  r&  r   r   )r  r&  r   r   )r   zituple[Any, list[Any], list[Any], Callable[[Any, Any], Any], Optional[dict[sympy.Symbol, pytree.KeyPath]]])NNF)r   zOptional[Sequence[int]]r  r#  r7  )r  r1  r2  )9rq   rr   rs   rt   r  ru   rH  rI  r~   r   r  r  r  r|   r  r  r  r  r  r  rC  rq  rt  r  r  r  r  r  r  r  rJ  r  r  r	  r"	  ry  r)	  r2	  r  r-	  r:	  r=	  r@	  r  rH	  rO	  rS	  rV	  r[	  r`	  re	  rm	  r  r  r  rv   r  r  s   @rx   r  r     sr   %'M?'(..tDFND-1K*1(,,%)O]) 4?3D3D4!= 
 	    6:N29<@9@<G<M<M=9  .9->->t-T*T &(, 
, ,<.!F!
*
";0
 
 
 k

k
 k
Z B
 B
H ! !@ ! !  *.6:Z 'Z 4	Z Zx 
 

 P P > > ? ? U U DP@4J:	$'>
& Hrw   r  c                  T   ^  \ rS rSrSS jr       S SU 4S jjjrSS jrSrU =r$ )	ExternKernelOuti  c                "   U R                  U5        / U R                  5       QU R                  SS9QnU R                  5       n[        R
                  R                  (       a  U R                  S:X  a  SnOU R                  5       nU R                  5       =n(       a  UR                  O[        R
                  R                  nUR                  UU R                  5       U R                  (       a  U R                  R                  5       OS UU5        g )NT)rY	  ztorch::inductor::_mm_plus_mmaoti_torch__mm_plus_mm_out)r  rS	  r[	  r  rV   r   r  r  r   r   r  generate_extern_kernel_outr  r  )r;  r  r   rt	  r  r   s         rx   r  ExternKernelOut.codegen  s    W%J""$Jt':':D':'IJ**,GG$$(FF 7K..0K!%!22A29L9L**""$484D4DD..0$	
rw   c
                   > [         T
U ]  S UU R                  U5      UU=(       d    0 S UUUU	5
        [        R                  R                  U 5      U l        [        R                  R                  U 5        g r   )r  rC  r  rV   r   r  r   r  )r;  r   r  r  r   r  r  r  r  r  r  s             rx   rC  ExternKernelOut.__init__#  si     	'Lb)	
 GG++D1		""4(rw   c                    gr  ro   rB  s    rx   rJ  ExternKernelOut.should_allocate>  r]  rw   rG  r  rv	  r   )	rq   rr   rs   rt   r  rC  rJ  rv   r  r  s   @rx   ry	  ry	    s=    
2 &() 
) )6 rw   ry	  c                  ,   ^  \ rS rSrSU 4S jjrSrU =r$ )RandomSeedsiB  c           	       > [         R                  " [         R                  5      n[        TU ]  [        U[         R                  U/S9/ UR                  UR                  U//SS[        R                  R                  S9  g )Nrw  zaten.randint.low_outzat::_ops::randint_low_out::call)r   r  r  r  r  r  )r   r  r  r  rC  r"  r  r  r  randintlow_out)r;  countr   limitsr  s       rx   rC  RandomSeeds.__init__C  sl    U[[)kkW
 !::vzzE7;5 >,, 	 	
rw   ro   )r	  r   r   r+  r   r   rq   rr   rs   rt   rC  rv   r  r  s   @rx   r	  r	  B  s    
 
rw   r	  c                  X   ^  \ rS rSrSS jr      S SU 4S jjjrS	S jrS rSrU =r	$ )
r  iV  c                   U R                  U5        / U R                  5       QU R                  5       Qn[        R                  R
                  R                  X5        [        U R                  [        5      (       a  U R                  U5        g g r   )r  rS	  r[	  rV   r   r  generate_extern_kernel_allocr{   r   r  r`	  r;  r  r   s      rx   r  ExternKernelAlloc.codegenW  sm    W%=""$=t':':'<=	99$Edkk6**%%g. +rw   c	                   > [         T	U ]  S UU R                  U5      UU=(       d    0 S UUUU5
        / U l        [        R
                  R                  U 5      U l        [        R
                  R                  U 5        g r   )	r  rC  r  r  rV   r   r  r   r  )
r;  r   r  r  r   r  r  r  r  r  s
            rx   rC  ExternKernelAlloc.__init__^  sp     	'Lb)	
 ')GG++D1		""4(rw   c                    gr  ro   rB  s    rx   rJ  !ExternKernelAlloc.should_allocate|  r  rw   c                    [         er   rU  rB  s    rx   r  "ExternKernelAlloc.apply_constraint  rW  rw   )r   r  r  )ro   NNNro   Nr   )
rq   rr   rs   rt   r  rC  rJ  r  rv   r  r  s   @rx   r  r  V  s?    / &() 
) )<" "rw   r  c                  J   ^  \ rS rSrSrSU 4S jjrS	S jrS rS
S jrSr	U =r
$ )r  i  zH
An output buffer that represents the mutation of a pre-existing buffer
c                   > [         TU ]  S US9  UR                  5       n[        R                  R                  U5        U/U l        X0l        [        R                  R                  U 5      U l	        g rb  )
r  rC  r  rV   r   r  mutation_namesmutating_noder  r   )r;  r   mutated_noder	  mutated_node_namer  s        rx   rC  MutationOutput.__init__  s^    d62(113	##$5601(5GG++D1	rw   c                    U R                   $ r   )r	  rB  s    rx   r[  MutationOutput.get_defining_op  s    !!!rw   c                    U R                   $ r   )r	  rB  s    rx   r  !MutationOutput.get_mutation_names  rV  rw   c                    gr  ro   rB  s    rx   rJ  MutationOutput.should_allocate  r  rw   )r	  r	  r   )r	  rO  r   r   rT  r   )rq   rr   rs   rt   r  rC  r[  r  rJ  rv   r  r  s   @rx   r  r    s!    2"# rw   r  c                     ^  \ rS rSr% Sr0 rS\S'   \ S	       S
S jj5       r S	         SU 4S jjjr	SS jr
SrU =r$ )TMADescriptori  a  
An IR node representing a host-side TMA descriptor in the Triton API
(the ones obtained via create_{1d,2d}_tma_descriptor calls). Mostly
useful for user-defined Triton kernels relying on host-side TMA; but
can, in principle, be used for Inductor's Triton templates, too.
zdict[Any, TMADescriptor]_CACHEc                    [        U5      X#U4nXPR                  ;  a  [        XX45      U R                  U'   U R                  U   $ r   )idr	  r	  )r  r%  ra  
block_dimselement_sizer}  s         rx   r  TMADescriptor.create  s@     &z4\:jj +F*SCJJsOzz#rw   c           
     H  > [        U5      S;   d   e[        U5      [        U5      :X  d   eUc  UR                  5       R                  nXl        X l        X0l        X@l        [        U R                  5      U l        U/n/ U R                  QU R
                  QU R                  Pn[        TU ]%  S [        [        UUR                  5       S95      U[        U5      S 5        [        R                  R!                  U 5      U l        [        R                  R%                  U 5        g )N)r1   r   r  )r   r   rB  r%  ra  r	  r	  r  r  rC  r  r$  r   r}   rV   r   r  r   r  )r;  r%  ra  r	  r	  r  r  r  s          rx   rC  TMADescriptor.__init__  s    4yF"""4yC
O+++!++-66L	$(		N	
YY
__
 
 	 !,,. - 	
  GG++D1		""4(rw   c                &    UR                  U 5        g r   )generate_tma_descriptorr  s     rx   r  TMADescriptor.codegen      ''-rw   )r	  ra  r	  r   r  r%  r   )r%  rc   ra  list[Union[int, torch.SymInt]]r	  r	  r	  r4  )
r%  rc   ra  r	  r	  r	  r	  r4  r   r   r  )rq   rr   rs   rt   r  r	  ru   r  r  rC  r  rv   r  r  s   @rx   r	  r	    s     (*F$) '+

 -
 3	

 $
 
" '++)+) -+) 3	+)
 $+) 
+) +)Z. .rw   r	  c                  n   ^  \ rS rSrS rS
S jrSU 4S jjrSS jr  S
U 4S jjrSS jr	SS jr
S	rU =r$ )UserDefinedTritonKerneli  c                z  ^ SSK Jn  SSKJn  UR	                  U R
                  5      m/ n/ n/ n[        TU5      (       a  [        TS5      (       a&  UR                  U4S jTR                   5       5        O.[        TS5      (       d   eUR                  TR                  5        [        TS5      (       a<  TR                   H+  nUR                  TR                  R                  U   5        M-     O.[        TS5      (       d   eUR                  TR                  5        TR                   nTR                  mTX4U4$ )	Nr   )	Autotuner)kernel_side_tablerestore_idxc              3  V   >#    U  H  nTR                   R                  U   v   M      g 7fr   )r   	arg_names)r  r   r  s     rx   r  BUserDefinedTritonKernel.get_kernel_and_metadata.<locals>.<genexpr>  s$      *4FqFII''*4Fs   &)restore_value	reset_idxreset_to_zero)triton.runtime.autotunerr	  *torch._higher_order_ops.triton_kernel_wrapr	  
get_kernel
kernel_idxr{   rZ  r  r	  r	  r	  r  r   r	  r	  configs)r;  r	  r	  r	  restore_value_argsreset_to_zero_argsr   r  s          @rx   get_kernel_and_metadata/UserDefinedTritonKernel.get_kernel_and_metadata  s   6P"--doo>(*(*fi(( v}--")) *4:4F4F*  v7777"))&*>*>?v{++))A&--fii.A.A!.DE * v7777"))&*>*>?nnGYYFw4FFFrw   c           
        SSK Jn  U R                  5       u  nnnnUR                  UUU R                  UUU R
                  5      u  nnn	U R                   V
s0 s H  oU R                  U
5      _M     nn
[        UR                   Vs/ s H  oR                  U   PM     sn5      n/ n/ n/ n[        R                  " UR                  5       [        [        R                  " S5      U	5      5       GHZ  u  nnUR!                  U5        [#        U[$        5      (       a@  UR!                  UR'                  5       5        UR!                  UR)                  5       5        Mm  [#        U[*        [,        [.        [0        R2                  45      (       a-  UR!                  U5        UR!                  [5        U5      5        M  UU;   a(  UR!                  S5        UR!                  [*        5        M  UcI   U" 5       (       a)  UR!                  S5        UR!                  [*        5        GM0  UR7                  5         GMC  [9        S[5        U5       SU 35      e   U R;                  U5        UR=                  UUUUUSU R?                  5       S9  g s  sn
f s  snf )	Nr   )triton_version_uses_attrs_dictr_  r  zUnsupported arg type: r   T)	arg_typesraw_argstriton_metar   r   ) torch._inductor.utilsr	  r	  !define_user_defined_triton_kernelr   gridr  rV	  r,   
constexprsr	  r  rz  r  r   repeatr  r{   rc   r  r   r   r  rm   r   r   r   r  rt  r  generate_kernel_callr   )r;  r  r	  r  r	  r	  r	  new_namer	  extra_launch_argsrZ	  
named_argsr   constexpr_namesr   r	  raw_args_filteredr   r  s                      rx   r  UserDefinedTritonKernel.codegen  sH   H ((*	
 55KKII
		
 261S1S
1SAt$$Q''1S 	 
 %6CTCT%UCTa&6&6q&9CT%UV!	')"I$4$4R$8:K L
ID# $$S)#v&&C1134  1C#udEJJ!?@@C   c+( B  % 233KKO$$S)%))+),B49+RPSu*UVV?
B 	W%$$&#??$ 	% 	
U
 &Vs   JJ	c                L   > [         TU ]  5       [        U R                  5      -  $ r   )r  r  r&   r	  r  s    rx   r  0UserDefinedTritonKernel.get_unbacked_symbol_usesO  s"     w/14I$))4TTTrw   c                    [        5       $ r   r+   rB  s    rx   rt  0UserDefinedTritonKernel.get_unbacked_symbol_defsT  rv  rw   c          	       > / n0 n/ nUR                  5        H  u  p[        U	[        5      (       aY  [        R	                  U R                  U	5      5      n
X;   a  [        R                  " U
/X8   Q76 n
UR                  U
5        XU'   Ms  UR                  U	5        XU'   M     [        U5      S:w  d   eUS   R                  5       U l        [        TU ]5  S [        U R                  S9U[        U5      U5        Xl        X l        U R%                  5       u  p  nUR&                   Vs/ s H  oU;   d  M
  UPM     snU l        SSKJn  [        U5      S:  a  US   R.                  O0 nU" U0 UEUE5       Vs/ s H  nUU   PM
     snU l        U R0                   Vs/ s H!  n[3        [        U R                  S9UU 5      PM#     snU l        [6        R8                  R;                  U 5        g s  snf s  snf s  snf )Nr   r  )identify_mutated_tensors)r  r{   rb   r  rz  ry  r	  r  r  r   r   r   r  rC  r  r}   r	  r	  r	  r	  r  r	  r	  r   mutable_argsr  r  rV   r   r  )r;  r	  r	  tma_descriptor_metadatakernel_argsr  r   r  rZ	  r  r   r  r	  r   r  r	  autotuned_kwargsr}  r  r  s                      rx   rC   UserDefinedTritonKernel.__init__W  s    %%'DA!Y'' 99$:L:LQ:OP/%,,QL1H1KLAa q	$$Q'q	 ( 6{aQi**,dkk*- 	
 %	 $ < < >A "++.
+Ck/AC+.
* 	X03Gq0@71:,,b 0;;;*:;
 
 ((!
( :T[[93E(!
 	
""4(%.

!
s   (	G-5G-8G2(G7c                ,    [        U R                  5      $ r   )r|   r  rB  s    rx   rq  #UserDefinedTritonKernel.get_outputs  s    D))**rw   c                    U R                   $ r   r  rB  s    rx   r   "UserDefinedTritonKernel.get_device  r  rw   )r   r	  r	  r	  r  r  r  r~  r|  r(  )rq   rr   rs   rt   r	  r  r  rt  rC  rq  r   rv   r  r  s   @rx   r	  r	    s8    G>I
VU
3)	3)j+ rw   r	  c                  T   ^  \ rS rSrSrS	S jrS
S jrS rSS jrS	U 4S jjr	Sr
U =r$ )InplaceBernoulliFallbacki  =
This needs to be a custom class to handle mutation properly
c                   S U R                    5       u  n[        R                  R                  (       a\  UR	                  U R                  5        SU SSR                  [        [        U R                  5      5       SUR                   35        g UR	                  U R                  5        SU SSR                  [        [        U R                  5      5       SUR                   35        g )Nc              3  @   #    U  H  oR                  5       v   M     g 7fr   r  r  r   s     rx   r  3InplaceBernoulliFallback.codegen.<locals>.<genexpr>  s     ;{!##%%{   rf  r  z, NULL)r   )r  rV   r   r  r  r  ri  rg  reprr  ending)r;  r  r   s      rx   r   InplaceBernoulliFallback.codegen  s    ;t{{;77 '')*!A3b3tTEWEW;X1Y0ZZabibpbpaqr '')*!A3b3tTEWEW;X1Y0ZZ[\c\j\j[klrw   c                    gr  ro   rB  s    rx   rJ  (InplaceBernoulliFallback.should_allocate  r  rw   c                >    U R                   S   R                  5       /$ r  r  r  rB  s    rx   r  +InplaceBernoulliFallback.get_mutation_names      A'')**rw   c                    [        5       $ r   r+   rB  s    rx   rt  1InplaceBernoulliFallback.get_unbacked_symbol_defs  rv  rw   c                R  > [         TU ]  S [        UR                  5       S9U R	                  U/5      UUS9  [
        R                  R                  UR                  5       5        [
        R                  R                  U 5      U l
        [
        R                  R                  U 5        g )Nr  r  )r  rC  r  r   r  rV   r   r  r  r  r   r  )r;  r  r   r  r  s       rx   rC  !InplaceBernoulliFallback.__init__  s~    alln-$# 	 	
 	
##AJJL1GG++D1		""4(rw   rG  r  r   r~  rq   rr   rs   rt   r  r  rJ  r  rt  rC  rv   r  r  s   @rx   r	  r	    s&    +
) 
)rw   r	  c                  p   ^  \ rS rSrSrS
S jrSS jrS rSS jr  S
U 4S jjr	\
SSS jj5       rS	rU =r$ )InplaceCopyFallbacki  r	  c                N    U R                  5       u  p#nUR                  X2U5        g r   )rS	  codegen_device_copy)r;  r  r  r  non_blockings        rx   r  InplaceCopyFallback.codegen  s%    #'#4#4#6 <##Cl;rw   c                    gr  ro   rB  s    rx   rJ  #InplaceCopyFallback.should_allocate  r  rw   c                >    U R                   S   R                  5       /$ r  r	  rB  s    rx   r  &InplaceCopyFallback.get_mutation_names  r	  rw   c                    [        5       $ r   r+   rB  s    rx   rt  ,InplaceCopyFallback.get_unbacked_symbol_defs  rv  rw   c           	       > [         TU ]  S UUUSSS9  [        R                  R	                  US   R                  5       5        [        R                  R                  U 5      U l        [        R                  R                  U 5        g )Nz
aten.copy_aoti_torch_copy_)r  r  r   )	r  rC  rV   r   r  r  r  r   r  )r;  r   r  r  r  s       rx   rC  InplaceCopyFallback.__init__  sr     	+. 	 	
 	
##F1I$6$6$89GG++D1		""4(rw   c                    X4 Vs/ s H  o@R                  U5      PM     nnU4n[        [        UR                  5       S9UU5      nU$ s  snf rn  )ry  r 
  r  r   )r  r  r  r
  r   r  r  r  s           rx   r  InplaceCopyFallback.create  sS    14
;
1##A&
;%$cnn./

  <s   A	rG  r  r   r~  r7  )r
  rm   )rq   rr   rs   rt   r  r  rJ  r  rt  rC  r  r  rv   r  r  s   @rx   r 
  r 
    s?    <+)
 
)$  rw   r 
  c                  F    \ rS rSrSrS
S jrSS jrS rSS jrSS jr	Sr
g	)MutatingFirstArgExternKerneli  r	  c                    / S U R                    5       Q[        [        U R                  5      QnUR	                  U R                  5        SSR                  U5       SUR                   35        g )Nc              3  @   #    U  H  oR                  5       v   M     g 7fr   r	  r	  s     rx   r  7MutatingFirstArgExternKernel.codegen.<locals>.<genexpr>  s     9[!!##[r	  rf  r  r   )r  rg  r	  r  r  r  ri  r	  )r;  r  argrefss      rx   r  $MutatingFirstArgExternKernel.codegen  sl    
9T[[9
t))*
 	##%&a		'(:';1W^^<LM	
rw   c                    gr  ro   rB  s    rx   rJ  ,MutatingFirstArgExternKernel.should_allocate  r  rw   c                >    U R                   S   R                  5       /$ r  r	  rB  s    rx   r  /MutatingFirstArgExternKernel.get_mutation_names  r	  rw   c                    [        5       $ r   r+   rB  s    rx   rt  5MutatingFirstArgExternKernel.get_unbacked_symbol_defs  rv  rw   c                    gr  ro   rB  s    rx   has_side_effects-MutatingFirstArgExternKernel.has_side_effects  r]  rw   ro   Nr  r   r~  )rq   rr   rs   rt   r  r  rJ  r  rt  r
  rv   ro   rw   rx   r
  r
    s     
+rw   r
  c                  ,   ^  \ rS rSrSU 4S jjrSrU =r$ )ResizeStorageBytesi  c                (  > [        U[        5      (       d   S5       e[        TU ]  S [	        UR                  5       S9U R                  U/5      U4S9  [        R                  R                  UR                  5       5        [        R                  R                  U 5      U l        [        R                  R                  U 5        SU l        SU l        [        R                  R                   R#                  UR$                  R                  5       5        g )NzTODO: dynamic shapesr  )r  z"inductor_ops.resize_storage_bytes_z&torch::inductor::resize_storage_bytes_)r{   r   r  rC  r  r   r  rV   r   r  r  r  r   r  r  r  never_reuse_buffersr  r  )r;  variablerP  r  s      rx   rC  ResizeStorageBytes.__init__	  s    (C((@*@@(h1134
+#+	 	 	
 	
##H$5$5$78GG++D1		""4("FG	##''(>(>(@Arw   )r  r   r  r  r	  r  s   @rx   r!
  r!
    s    B Brw   r!
  c                  6   ^  \ rS rSrSU 4S jjrSS jrSrU =r$ )SetSourceTensorKerneli  c                  > UR                  5         [        TU ]	  UR                  5       X/S[        R
                  R                  R                  R                  S9  [        R                  R                  R                  UR                  R                  5       5        [        R                  R                  R                  UR                  5       5        [        R                  R                  R                  U R                  5       5        UR                  5       n[!        [#        US9X5      [!        [#        US9X 5      /U l        g )Nz!torch.ops.aten.set_.source_Tensor)r  r  r  )r  r  rC  r   r   rT   r  set_source_TensorrV   r   r#
  r  r  r  r   r  r  r  )r;  self_tensorstorage_tensorr   r  s       rx   rC  SetSourceTensorKernel.__init__  s    $$&%%')B		++99	 	 	
 	
##''(8(8(A(A(CD	##''(?(?(AB	##''8**,:V4kH:V4nK!
rw   c                v    U R                   S   R                  5       U R                   S   R                  5       /$ ro  r	  rB  s    rx   r  2SetSourceTensorKernel.get_inputs_that_alias_output+  s/    A'')4;;q>+B+B+DEErw   r  r  rG  )rq   rr   rs   rt   rC  r  rv   r  r  s   @rx   r'
  r'
    s    
"F Frw   r'
  c                  l   ^  \ rS rSrSrSS jrSS jrS rSS jrSSS	.       SU 4S
 jjjr	Sr
U =r$ )ScatterFallbacki/  z
This needs to be a custom class to handle mutation properly.
This class handles both aten.scatter_ and aten.scatter_reduce_.
It also handle the case `src` being a scalar properly.
c           
        U R                   S   n[        R                  R                  (       a  SSS.nX#;   a  X2   nU R                  (       a  S U R
                   5       u  pEnO$S U R
                   5       u  pEU R                  S   nUR                  UX@R                  S   XV/U R                  U R                  U R                  UU R                  5       5        g )	Nr  r  r  )r  multiplyc              3  @   #    U  H  oR                  5       v   M     g 7fr   r	  r	  s     rx   r  *ScatterFallback.codegen.<locals>.<genexpr>?  s     Jk2244kr	  c              3  @   #    U  H  oR                  5       v   M     g 7fr   r	  r	  s     rx   r  r5
  A  s     EA--//r	  r1   r   )r   rV   r   r  src_is_tensorr  r  generate_scatter_fallbackr  r  r[	  )r;  r  r  get_operator_enumr   r   r  s          rx   r  ScatterFallback.codegen6  s    X&77(-6 B**2JdkkJOQsEEJQ$$Q'C))""1%u2  ##!	
rw   c                    gr  ro   rB  s    rx   rJ  ScatterFallback.should_allocateM  r  rw   c                >    U R                   S   R                  5       /$ r  r	  rB  s    rx   r  "ScatterFallback.get_mutation_namesP  r	  rw   c                    [        5       $ r   r+   rB  s    rx   rt  (ScatterFallback.get_unbacked_symbol_defsS  rv  rw   NTr  include_selfc               d  > [        U[        5      U l        U R                  (       a&  X$U4 Vs/ s H  oR                  U5      PM     n	nU4n
O$X$4 Vs/ s H  oR                  U5      PM     n	nX54n
[        TU ]  S [        UR                  5       S9U R                  U	5      U
XgS.[        U5      SS/US9  [        R                  R                  UR                  5       5        [        R                  R                  U 5      U l        [        R                  R!                  U 5        g s  snf s  snf )Nr  rA
  r  rB
  )r  r  r  )r{   rb   r7
  ry  r  rC  r  r   r  r   rV   r   r  r  r  r   r  )r;  r  r   r  r   r  r  rB
  r   tensorsr  r  s              rx   rC  ScatterFallback.__init__V  s    (Y7 78oFo))!,oGF FM78jAj))!,jGA JMalln-(<";/+3^*D# 	 		
 	
##AJJL1GG++D1		""4(% G Bs   D(D-)r   r7
  r  r   r~  )r  r   r  r&  rB
  rm   r   r   r	  r  s   @rx   r1
  r1
  /  sV    
.+ !%!!) 	!) !) !) 
!) !)rw   r1
  c                  T   ^  \ rS rSrSrS	S jrS
S jrS rSS jrS	U 4S jjr	Sr
U =r$ )IndexPutFallbackiz  zI
This needs to be a custom class to handle mutation and indices properly
c                   S U R                    5       tp#n/ n[        U5      n[        U R                  5       Hd  u  pxU R                  U   b  UR	                  [        U5      5        M1  UR	                  [        R                  R                  R                  5        Mf     UR                  " U R                  5       X%U/U R                  5       Q76   g )Nc              3  @   #    U  H  oR                  5       v   M     g 7fr   r	  r	  s     rx   r  +IndexPutFallback.codegen.<locals>.<genexpr>  s     &Rk':':'<'<kr	  )r  r  r   r  r  r  rV   r   r  rk  generate_index_put_fallbackr  rO	  )	r;  r  r   r   valid_indicesr  iter_valid_indicesr   r   s	            rx   r  IndexPutFallback.codegen  s    &Rdkk&R#]!-0dll+DA||A*t$678qww33<<=	 , 	++  "A	
9=9P9P9R	
rw   c                    gr  ro   rB  s    rx   rJ   IndexPutFallback.should_allocate  r  rw   c                >    U R                   S   R                  5       /$ r  r	  rB  s    rx   r  #IndexPutFallback.get_mutation_names  r	  rw   c                    [        5       $ r   r+   rB  s    rx   rt  )IndexPutFallback.get_unbacked_symbol_defs  rv  rw   c           
       > X0l         U Vs/ s H	  ofc  M  UPM     nnX$/UQ Vs/ s H  o R                  U5      PM     nnSn	[        T
U ]  S [	        WR                  5       S9U R                  U5      U4SU	US9  [        R                  R                  U R                  S   R                  5       5        [        R                  R                  U 5      U l        [        R                  R                  U 5        g s  snf s  snf )Naoti_torch_index_put_outr  zaten.index_put_)r  r  r  r   )r  ry  r  rC  r  r   r  rV   r   r  r  r  r  r   r  )r;  r  r   r  r   
accumulater   rL
  rD
  r  r  s             rx   rC  IndexPutFallback.__init__  s    $+=GqG=342M}2MN2MQ%%a(2MN4alln-(M0+# 	 	
 	
##DKKN$;$;$=>GG++D1		""4( >Ns   C9C9C>r  r   r  r   r~  r	  r  s   @rx   rG
  rG
  z  s&    
+) )rw   rG
  c                  .    \ rS rSr\S 5       rSS jrSrg)
DeviceCopyi  c                   UR                  5       (       dU  [        S UR                  5        5       5      (       a0  [        R                  R
                  (       d  UR                  U5      $ [        R                  R                  U5        [        R                  R                  UR                  5       5        [        S5        U4n[        [        UUR                  5       UR                  5       S9U R!                  U5      /U5      $ )Nc              3  Z   #    U  H!  o[         R                  R                  ;   v   M#     g 7fr   )rV   r   r	  r  s     rx   r  $DeviceCopy.create.<locals>.<genexpr>  s     G4Fq***4Fs   )+zDeviceCopy in input programrw  )r  r	  rO  r2   aot_inductoruse_runtime_constant_foldingr  rV   r   add_device_infor   rJ   r[
  r   r   r   ry  )r  r   r   r
  r  s        rx   r  DeviceCopy.create  s     GA4D4D4FGGG''DD''//	'	/78%kkmZZ\
 q!"
 	
rw   c                   U R                  5       n[        U5      S:X  d   eU R                  (       a2  UR                  US   U R                  R	                  5       US   5        g UR                  US   U R	                  5       US   5        g )Nr   r   r1   )rS	  r   r  r
  r  r	  s      rx   r  DeviceCopy.codegen  s{      "4yA~~''Q));;=tAw ''Q1G1G1I4PQ7Srw   ro   Nr  )rq   rr   rs   rt   r  r  r  rv   ro   rw   rx   r[
  r[
    s    
 
.Trw   r[
  c                  X   ^  \ rS rSrSrS	S jrS
S jrSU 4S jjrSS jrSS jr	Sr
U =r$ )r   i  z3
The result of a call to aten._local_scalar_dense.
c                    [        5       $ r   r+   rB  s    rx   rN  DynamicScalar.get_reads  rv  rw   c                    gr  ro   rB  s    rx   rJ  DynamicScalar.should_allocate  r  rw   c                   > UR                  5         [        TU ]	  S [        [        R
                  " S5      S9U R                  U/5      5        Xl        X l        g Nr   r  )	r  r  rC  r  r   r   r  symkeypath)r;  rl
  rm
  r  r  s       rx   rC  DynamicScalar.__init__  sG    *ELL$78$:M:Mtf:U	
 rw   c                .    [        U R                  /5      $ r   )r,   rl
  rB  s    rx   rt  &DynamicScalar.get_unbacked_symbol_defs  s    488*%%rw   c                &    UR                  U 5        g r   )codegen_dynamic_scalarr  s     rx   r  DynamicScalar.codegen  s    &&t,rw   )rm
  rl
  r?  r   r  r~  )rq   rr   rs   rt   r  rN  rJ  rC  rt  r  rv   r  r  s   @rx   r   r     s&    &- -rw   r   c                  ^   ^  \ rS rSrSrS
S jrSS jrSU 4S jjrSS jrS r	SS jr
S	rU =r$ )r   i  z-
The result of a call to aten._assert_scalar
c                    [        5       $ r   r+   rB  s    rx   rN  AssertScalar.get_reads  rv  rw   c                    gr  ro   rB  s    rx   rJ  AssertScalar.should_allocate  r  rw   c                v   > [         TU ]  S [        [        R                  " S5      S9/ 5        Xl        X l        g rk
  )r  rC  r  r   r   scalarr	  )r;  rz
  r	  r  s      rx   rC  AssertScalar.__init__  s3    ell512	
 rw   c                    gr  ro   rB  s    rx   r
  AssertScalar.has_side_effects  r]  rw   c                ,    [        U R                  5      $ r   )r&   rz
  rB  s    rx   r  %AssertScalar.get_unbacked_symbol_uses   s    $T[[11rw   c           	        [         R                  (       d  g [        [        U R	                  5       5      5      n[
        R                  R                  (       a^  SU S3n[
        R                  R                  R                  U R                  SS9nUR                  SU SU R                   SU S35        g [
        R                  R                  R                  U R                  SS9nUR                  S	U S
35        UR                  S[        U R                  5       S35        UR                  U R                  5        S35        g )Nzstd::to_string(r   F)r  zif (!(z()) { throw std::runtime_error("Expected z but received " + z); }zif not rR  z    raise RuntimeError(z = None)r2   scalar_assertsr  r  r  rV   r   r  r  codegen_cpp_sizevarrz
  r  r	  codegen_python_sizevarr	  r  )r;  r  symbol
symbol_strsizevars        rx   r  AssertScalar.codegen  s   $$ d488:;<77*6(!4Jgg**>>e ? G 	!J488*Tfgqfrrwx gg**AAe B G y23 7TXX7GqIJ  19:rw   )r	  rz
  r?  r   r  )rq   rr   rs   rt   r  rN  rJ  rC  r
  r  r  rv   r  r  s   @rx   r   r     s+    	2; ;rw   r   c                  *    \ rS rSr% S\S'   S\S'   Srg)ExternKernelNodei   r   r   zexport_schema.Noder   ro   Nrp   ro   rw   rx   r
  r
     s    
I
rw   r
  c                     ^  \ rS rSr SSS. SU 4S jjjjrSS jrSS jrS r\S 5       r	S	 r
S
 rS rS rSS jr\SS j5       r\S 5       rU 4S jrSrU =r$ )FallbackKerneli&  Nr  c                 >^  [         TT ]  U[        U5      [        U5      US9  ST l        UT l        [        U[        R                  R                  [        R                  R                  45      (       d   SU S[        U5       S35       eUT l        UT l        Uc  0 OUT l        [        R                  R!                  T R"                  5        / T l        / T l        [        T R                  [        R                  R                  5      (       a  g ST R                  R)                  5       ;   a  g T R                  R*                  n[        R,                  R.                  R1                  T R                  5      (       a-  T R&                  R3                  US   R5                  5       5        g UR6                  (       a  [9        U5      (       d  [;        SU 35      eT R                  T R<                  T R>                  5      u  pS
U 4S	 jjn
[        R,                  R.                  RA                  XU5       H  u  pU
" X5        M     g )Nr	  Fz#Fails to create FallbackKernel for r   z not supported_c10d_functionalr   z'NYI: Can't generate FallbackKernel for c                "  >^  [        T R                  [        R                  5      (       a  [        U[        [
        45      (       d   e[        R                  " T R                  5      (       a  [        U[
        [        45      (       a   eUc  g T R                  c  g SU U4S jjn[        R                  " T R                  5      (       a  Ub  U H  nU" U5        M     g g [        R                  " T R                  5      (       d   eU" U5        g )Nc                  > TR                   R                  U R                  5       5        TR                  R                  (       a<  TR
                  R                  [        [        U R                  5       S9U T5      5        g g rn  )	alias_namesr  r  
alias_infois_writer  r  r  r   )r   infor;  s    rx   	add_aliasPFallbackKernel.__init__.<locals>.handle_aliasing_and_mutation.<locals>.add_alias  sZ      ''

5??++))00&z'H!TR ,rw   r  )
r{   r   r   ListTyper|   r}   library_utilsis_tensor_like_typer
  is_tensorlist_like_type)r
  r  r
  optional_tensor_argr;  s   `   rx   handle_aliasing_and_mutation=FallbackKernel.__init__.<locals>.handle_aliasing_and_mutations  s    $))U^^44!#e}555500;; &cE4=9999{&  44TYY???/2+!"56 03 # %88CCCC#rw   r  )!r  rC  r}   use_runtime_dispatchr  r{   r   r  r  r  r   r  r  r   rV   r   warn_fallbackr  r
  r	  r   r  _libraryr  mutates_and_returns_first_argr  r  
is_mutabler   rt  r  r  
zip_schema)r;  r   r  r	  nontensor_argsr  r   r  schemar   r
  r
  r  r  s   `            rx   rC  FallbackKernel.__init__'  s    	+.!	 	 	
 %*!!2

%%

..
 
 	X 14<.W	X 
 ","Nb	d556 '))+d&&

(F(FGG !1!1!6!6!88
 !!)) >>==d>N>NOO&&{1~'>'>'@A%;F%C%C%9&B  **4;;8J8JK	: --88vNID(3 Orw   c           	     n    UR                  U R                  5       U R                  [        U SS 5      5      $ Nr  )(codegen_unbacked_symbol_defs_for_outputsr  r  r   r  s     rx   codegen_unbacked_symbol_defs+FallbackKernel.codegen_unbacked_symbol_defs  s0    ??MMOT\\749Ld+S
 	
rw   c                    [        U SS 5      =n(       aC  [        [        R                  R                  R
                  U5      nUc   eUR                  5       $ [        5       $ r
  r   r(   rV   r   r   r   r  r,   r;  r  resolveds      rx   rt  'FallbackKernel.get_unbacked_symbol_defs  Z     '.A4 HHH0  **,=H '''==?"<rw   c                F   [         R                   " S S5      5       nU R                   Vs/ s H  o!" UR                  5       5      PM     nnU R	                  X0R
                  5      u  pE[        R                  R                  (       a  [        U R                  [        R                  R                  5      (       a  U R                  XE5      n[        U R                  R                   R"                  U5       VVs/ s H8  u  pb[        R                  R$                  R'                  X&R(                  5      PM:     nnnO9U Vs/ s H,  n[        R                  R$                  R'                  U5      PM.     nnU R*                  R-                  U5        U$ s  snf s  snnf s  snf )Nc                  *    \ rS rSr% S\S'   SS jrSrg))FallbackKernel.codegen_args.<locals>.Shimi  r   refc                    U R                   $ r   )r
  rB  s    rx   r  2FallbackKernel.codegen_args.<locals>.Shim.__repr__  s    xxrw   ro   Nr2  )rq   rr   rs   rt   ru   r  rv   ro   rw   rx   Shimr
    s    H rw   r
  )rH  	dataclassr  r  r  r  rV   r   r  r{   r  r   r  r  rH	  r   r  r  r  rK	  r  r   rr  )r;  r
  r   r	  r   r   params          rx   rS	  FallbackKernel.codegen_args  sH   				  	  
	  =AKKHKqtA//12KH**;8J8JK77:d.>.>

@U@U#V#V..t<D !$D$4$4$<$<$F$F M MHE $$33AG M  D
 EIIDqAGG((77:DDI 	6" I
 Js   F:?F3Fc                    U (       a;  U  Vs/ s H)  o"R                  5       (       d  M  UR                  5       PM+     nnUS   $ [        U[        R                  5      (       a  UR                  $ [        U[
        [        45      (       al  [        S U 5       5      nU Vs/ s H  oU(       d  M  UPM     nn[        U5      S:X  a  US   $ U H!  n[        UR                  5      (       d  M  Us  $    US   $ g s  snf s  snf )Nr   c              3  N   #    U  H  n[         R                  S U5      v   M     g 7fr   )r
  find_devicer  s     rx   r  -FallbackKernel.find_device.<locals>.<genexpr>  s#      $=K**433^rO  r1   )r   r{   r   r	  r   r|   r}   r,   r   rN   r   )r	  r	  r  devices
device_setr   s         rx   r
  FallbackKernel.find_device  s    3>S;C..BR's~~';GS1:nell33!(((ntUm44# $=K$ J -7AJ&&vJGA7|q qz!!&++&&!M " 1:! T Bs   C6C6
C;-C;c                    [        U R                  [        R                  R                  5      (       a  g[        U R                  5      R                  5       $ r  )r{   r  r   r  r  r$   r
  rB  s    rx   r
  FallbackKernel.has_side_effects  s<    d&&

(F(FGGt//0;;==rw   c                    U R                   $ r   )r
  rB  s    rx   r  +FallbackKernel.get_inputs_that_alias_output  rY  rw   c                P    [        U R                  5      S::  d   eU R                  $ r  )r   r	  rB  s    rx   r  !FallbackKernel.get_mutation_names  s'    4&&'1,,,"""rw   c           
        [         R                  SU R                  5       U R                  5        [	        U [
        5      (       d   eU R                  U R                  U R                  5      u  pU R                  X5      nU R                   Vs/ s H  nU R                  " U40 UD6PM     nnU R                  n[        R                  R                  (       d  / UQUQ$ [        S S 5      nUR!                  XQU5      nS n[	        U["        R$                  R&                  R(                  5      (       a#  UR+                  US   US   5      R,                  n	OUR.                  R,                  n	[1        U	5      S:X  aB  U R2                  (       a  U R2                  OU R4                  n
U	S   R6                  nU" X5      /nO:[9        XR2                  5       VVs/ s H  u  pU" UR6                  U5      PM     nnn[;        U R                  5       [<        R>                  " U R                  RA                  5       UU0 S9S9n[        R                  RB                  RE                  U5        / UQUQ$ s  snf s  snnf )Nz4Extern kernel node added for node %s with target %s.c           	        [        U [        R                  5      (       ap  Un[        U[        [        45      (       a  [        U5      S:X  d   eUS   n[        R                  R                  [        R                  " UR                  5       S9S9$ [        U [        R                  5      (       a{  [        U R                  5       [        R                  5      (       aN  [        R                  R                  U Vs/ s H%  n[        R                  " UR                  5       S9PM'     snS9$ [        S[        U 5       35      es  snf )Nr1   r   rG  )	as_tensor)
as_tensorszUnsupported return type )r{   r   
TensorTyper|   r}   r   export_schemaArgumentr  TensorArgumentr  r
  getElementTypeRuntimeErrorr   )return_typeoutputr   s      rx   handle_single_outputFFallbackKernel.export_extern_kernel_node.<locals>.handle_single_output  s   +u'7'788ftUm44v;!+++ )C$--44+::O 5   K88Z**,e.>.>> > %--44 $* #)C &44#,,.I#)  5   #%=d;>O=P#QRR s   9,Er   r1   )r]  r  r  metadata)r   r   )#r  r  r  r  r{   r
  r  r  r  rH	  r  rV	  rV   r   aot_moder   serialize_inputsr   _higher_order_ops	torchbindCallTorchBindr
  returnsr  r   r  r  r  r   r
  r
  rW   r   extern_kernel_nodesr  )r;  r   r   r}  ordered_kwargsr]  
serializernamed_argumentsr
  r
  r  r
  output_argumentsreturn_schemar
  r   s                   rx   export_extern_kernel_node(FallbackKernel.export_extern_kernel_node  s+   		BMMO	
 $////**4;;8J8JK**48 99
9 !!#009 	 
 !!ww+T+N++*46
$55fFK	S. fe55??MMNNmmDGT!W5==Gnn,,Gw<1 '+lldll8M8MG!!*..K 4[ JK .1,,-G -G)M %]%<%<fE-G   
  ##'',,.&(	
 	
##**40''''G
d s   I*I/c                  ^ ^^ T R                   nUR                  S:X  a}  [        U[        R                  R
                  5      (       d   e[        R                  R                  (       a2  SSK	J
n  [        U5      U;  a  [        R                  SU5        ST l        ObUR                  S:X  a,  [        U[        R                  R
                  5      (       d   eO&[        R                  R                  (       a  ST l        S
U U4S jjnSS jmT R                  T5        T R                  (       a	  U" 5         GO/ T R!                  5       QT R#                  5       Qn[        R                  R                  (       ah  [        U[        R                  R
                  5      (       a?  [%        U4S	 j['        XRR(                  R*                  5       5       5      (       a  U" 5         OZ[        R                  R,                  R/                  T U5        [        T R0                  [2        5      (       a  T R5                  T5        T R7                  T5        g )Nr  r   )inductor_fallback_opszG%s is missing a c-shim implementation, using proxy executor as fallbackT
_quantizedc            	       > S n TR                  5       nTR                  TR                  5       TR                  TR                  U TR
                  UTR                  (       a  TR                  5        g TR                  5        g r   )r
  ,generate_fallback_kernel_with_runtime_lookupr  r  r  r  r  r  )r   exported_argsr;  r  s     rx   do_runtime_dispatch3FallbackKernel.codegen.<locals>.do_runtime_dispatchF  sk    D ::<M@@''$$   $	 372G2G	rw   c                    [        U [        R                  5      =(       dI    [        U [        R                  5      =(       a(    [        U R	                  5       [        R                  5      $ r   )r{   r   
NumberTypeOptionalTyper
  )r   s    rx   	is_number)FallbackKernel.codegen.<locals>.is_numberU  sK    a!1!12 1e001 Eq//153C3CDrw   c              3  d   >#    U  H%  u  pS U;   =(       a    T" UR                   5      v   M'     g7f)zc10::complexN)r  )r  arg_strop_argr
  s      rx   r  )FallbackKernel.codegen.<locals>.<genexpr>c  s2      +N #g-M)F<L<L2MM+Ns   -0r  )r   ztorch.JitTyper   rm   )r  r  r{   r   r  r  rV   r   r  torchgen.aoti.fallback_opsr
  r   r  r  r
  r  rS	  r[	  r  r   r  r  r  generate_fallback_kernelr   r  r`	  r
  )r;  r  r  r
  r
  r   r
  s   ``    @rx   r  FallbackKernel.codegen/  s   !!v%fejj&;&;<<<<ww""Lv;&;; KKa 15D--fejj&;&;<<<<WW  (,D%	 		 	W%$$!AT&&(A4+>+>+@AD##vuzz'<'<== +.t^^5M5M+N   $%$$==dDIdkk622--g6))'2rw   c           	         [        U R                  U R                  [        U R	                  5       5      [        U R                  5       5      5      $ r   )r"  r   r   rH   r   r   )r
  s    rx   tensor_to_layoutFallbackKernel.tensor_to_layoutu  s9    MMLL%fkkm4%fmmo6	
 	
rw   c           	       ^ ^^ [         R                  4nX;  a  [        R                  R                  O	[        5       nU   T R                  " U/UQ70 UD6u  nnnn	n
S S S 5        T R                  WW5      nUc  T " [        US9UUWW	W
S9mO U(       d   S5       eT " [        US9UUWW	W
S9mU UU4S jmT" U/ 5      n[        U[        [        [        45      (       a	  UTl        U$ U/Tl        U$ ! , (       d  f       N= f)Nr  r
  z"Not sure where to find device infoc                  >^ ^ [        T [        [        45      (       a/  [        T 5      " UUU 4S j[	        [        T 5      5       5       5      $ [        T [        5      (       a<  T R                  5        VVs0 s H  u  p#UT" UT[        T 5      U4/-   5      _M      snn$ [        T [        R                  5      (       a  [        TR                  T 5      TT5      $ [        T [        5      (       a  T $ [        T [        R                  5      (       a  T R                  R                  $ T b   S[        T 5       S35       eg s  snnf )Nc              3  Z   >#    U  H   nT" TU   T[        T5      U4/-   5      v   M"     g 7fr   )r   )r  r   generate_outputr  r
  s     rx   r  AFallbackKernel.create.<locals>.generate_output.<locals>.<genexpr>  s7      $/ $F1Iw4<:K9L/LMM/s   (+zFallbackKernel output type z is not supported)r{   r|   r}   r   r   r   r~   r  r   r	  MultiOutputr
  r   SymIntr   rl  )r
  r  r}  r  r  r   packeds   ``  rx   r   .FallbackKernel.create.<locals>.generate_output  s&   &4-00F| $"3v;/$   FD)) %+LLN$2 g$v,9L8M.MNN$2  FELL11"((0 
 FC((FELL11{{'''~ 1$v,?PQ~ %s   6%D>)r  *_fused_moving_avg_obs_fq_helper_functionalrV   r   r
	  r   r	  r
  r  rg  r{   r|   r}   r~   r  )r  r  r   r   fake_incorrect_kernelscontextr	  r	  r	  r  r  r   r  r   r  s   `            @@rx   r  FallbackKernel.create~  s   "&"Q"Q!S!'!EAGG;= 	  ""6;D;F;!  n=!&)"3F ???6!0"3F	6 "."5geT233$FN  &YFN Ws    C11
C?c                    > [         TU ]  5       $ r   )r  r  r  s    rx   r  FallbackKernel.apply_constraint  s    w'))rw   )r
  r   r	  r  r  r  r
  r   r  r~  )r
  torch.Tensor)rq   rr   rs   rt   rC  r
  rt  rS	  rJ  r
  r
  r  r  r
  r  r
  r  r  r  rv   r  r  s   @rx   r
  r
  &  s     j4 j4 
j4 j4X

 .  (>
 #M(^D3L 
 
 D DL* *rw   r
  c                  N   ^  \ rS rSrSrS	S jrS
S jrSS. SU 4S jjjrSrU =r	$ )ComplexViewi  z9View a complex number as two dtyped numbers or vice versac                    gr  ro   rB  s    rx   rJ  ComplexView.should_allocate  r  rw   c                >    U R                   S   R                  5       /$ r  r	  rB  s    rx   r  (ComplexView.get_inputs_that_alias_output  s    A'')**rw   Nr
  c          	     *   > [         TU ]  UUUUUUS9  g )Nr
  )r  rC  )r;  r   r  r	  r
  r  r  r  s          rx   rC  ComplexView.__init__  s)     	/ 	 	
rw   ro   r   rG  r  )
rq   rr   rs   rt   r  rJ  r  rC  rv   r  r  s   @rx   r  r    s)    C+ 
 

 
rw   r  c                  *    \ rS rSr% S\S'   SS jrSrg)rg  i  r+  r   c                    U R                   $ r   r  rB  s    rx   r   MultiOutputLayout.get_device  r  rw   ro   Nr(  )rq   rr   rs   rt   ru   r   rv   ro   rw   rx   rg  rg    s    rw   rg  c                  f   ^  \ rS rSrS rS	S jr      S
U 4S jjrSS jrSS jrSS jr	Sr
U =r$ )r  i  c                   [        U5      S:  a  US   u  p4[        U[        5      (       a  U R                  U SU S3USS  5      $ [        U[        5      (       aU  [
        R                  R                  R                  XR                  5       [        U5      5      nU R                  XRSS  5      $ [        U[        5      (       a  U R                  U SU S3USS  5      $ [        SU5      eU$ )Nr   []r1   z['z']znon supported index type: )r   r   r|   codegen_list_tuple_accessr}   rV   r   r  codegen_tuple_accessr  r   r~   r{  )r;  basenamer  ityper   tuple_accesss         rx   r  %MultiOutput.codegen_list_tuple_access  s    w<!qzHE%&&55
!A3a6H'RSRT+VVE5)) ww33HHmmos1v  55lABKPPE4((55
"QCr6JGTUTVKXX$%A5IIOrw   c                    UR                  U R                  5       U R                  U R                  S   R                  5       U R                  5      5        U R                  U5        g r  )codegen_multi_outputr  r  r  r  r`	  r  s     rx   r  MultiOutput.codegen  sN    $$MMO**4;;q>+B+B+DdllS	
 	!!'*rw   c                   > [         TU ]  S X/S5        [        R                  R	                  U 5      U l        [        R                  R                  U 5        X0l        g r  )r  rC  rV   r   r  r   r  r  )r;  r   rt  r  r  s       rx   rC  MultiOutput.__init__  sE     	vw3GG++D1		""4(rw   c                <    U R                   S   R                  5       $ r  )r  r  rB  s    rx   r  $MultiOutput.get_unbacked_symbol_uses  s    {{1~6688rw   c                |    [        U R                  5      S:X  a#  [        U R                  S   [        5      (       a  gg)Nr1   r   TF)r   r  r{   rc  rB  s    rx   rJ  MultiOutput.should_allocate  s/    t{{q t{{1~'899rw   c                    U R                    Vs/ s HI  n[        U[        5      (       d  M  [        UR	                  5       5      S:  d  M9  UR                  5       PMK     sn$ s  snf r  )r  r{   r
  r   r  r  )r;  r  s     rx   r  (MultiOutput.get_inputs_that_alias_output"  s\     {{
"#~.  C4467!; CLLN"
 	
 
s   A"A"A"rY
  r  )r   r   r  zlist[tuple[Any, ...]]r   r   r~  r   rG  )rq   rr   rs   rt   r  r  rC  r  rJ  r  rv   r  r  s   @rx   r  r    sB    $+		 '		
 
	9
 
rw   r  c                     \ rS rSr% SrS\S'   S-S jrS.S jrS/S jrS0S jr	S1S	 jr
S2S
 jrS3S4S jjrS5S jrS6S jrS7S jrS6S jr S8     S9S jjrS:S jrS;S jr S8     S<S jjrS=S jrS>S jrS?S jrS@S jrSAS jrSBS jrS-S jrS-S jrSCS jrSDS jrS2S jrSDS  jr SAS! jr!SES" jr"SFS# jr#SGS$ jr$S3SHS% jjr%\&SIS& j5       r'SJS' jr(SIS( jr)SKS) jr*\&S* 5       r+S2S+ jr,\,r-S,r.g)Lr  i-  z;
TensorBox / StorageBox allow in-place mutation of Tensors
rc   r  c                6    U R                   R                  5       $ r   r(  rB  s    rx   r  !MutableBox.has_exceeded_max_reads5  r*  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r   MutableBox.get_device8  r  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  MutableBox.make_loader;      yy$$&&rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.make_indexer>  r2  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.get_strideA  r  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  MutableBox.get_nameD  r  rw   Nc                8    U R                   R                  U5      $ r   )r  r  r  s     rx   r  MutableBox.has_large_inner_fnG  s    yy++I66rw   c                8    U R                   R                  U5      $ r   r$  r  s     rx   r  MutableBox.mark_reuseJ  r&  rw   c                6    U R                   R                  5       $ r   r0  rB  s    rx   r  MutableBox.realize_hintM  r2  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.unwrap_viewP  r4  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.freeze_layoutS  s    yy&&((rw   c                8    U R                   R                  X5      $ r   )r  r  r  s      rx   r  *MutableBox.freeze_layout_with_stride_orderV  s     yy88NNrw   c                8    U R                   R                  U5      $ r   )r  r  r  s     rx   r  (MutableBox.freeze_layout_with_fill_order[  s    yy66u==rw   c                8    U R                   R                  U5      $ r   )r  r  r  s     rx   r  (MutableBox.freeze_layout_with_same_order^  s    yy66v>>rw   c                8    U R                   R                  X5      $ r   )r  r  r  s      rx   r  +MutableBox.freeze_layout_with_exact_stridesa  s     yy99-WWrw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.get_read_writesf      yy((**rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   rN  MutableBox.get_readsi  r:  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  MutableBox.num_readsl  r:  rw   c                6    U R                   R                  5       $ r   r4  rB  s    rx   r  MutableBox.get_storage_numelo  r6  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  MutableBox.get_reduction_typer  r  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  MutableBox.get_reduction_sizeu  r  rw   c                6    U R                   R                  5       $ r   r8  rB  s    rx   r  MutableBox.is_externx  r:  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.is_no_op{  r  rw   c                8    U R                   R                  U5      $ r   r  r  s     rx   r  MutableBox.constant_to_device~  s    yy++F33rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.get_mutation_names  r  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  MutableBox.get_operation_name  r  rw   c                6    U R                   R                  5       $ r   )r  r  rB  s    rx   r  'MutableBox.get_inputs_that_alias_output  s    yy5577rw   c                6    U R                   R                  5       $ r   r,  rB  s    rx   r  MutableBox.realize  r.  rw   c                6    U R                   R                  5       $ r   r   rB  s    rx   r  #MutableBox.get_unbacked_symbol_uses  r  rw   c                6    U R                   R                  5       $ r   r@  rB  s    rx   rO  MutableBox.get_read_names  rB  rw   c                6    U R                   R                  5       $ r   )r  r[  rB  s    rx   r[  MutableBox.get_defining_op  rO  rw   c                8    U R                   R                  U5      $ r   )r  r  r  s     rx   r  MutableBox.codegen_reference  s    yy**622rw   c                6    U R                   R                  5       $ r   r  r  rB  s    rx   r   MutableBox.layout  s     yy((**rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r   MutableBox.get_layout  r  rw   c                6    U R                   R                  5       $ r   rq  rB  s    rx   r  MutableBox.get_output_spec  rO  rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r   MutableBox.get_size  r  rw   c                .    U R                   R                  $ r   )r  r   rB  s    rx   r   MutableBox.dtype  s    yyrw   c                ~   [        U R                  [        5      (       aQ  [        U 5      R                   S[        U R                  5      R                   S3nSnU R                  R                  nO&[        U 5      R                   S3nU R                  nSnU[        [        U5      5      U/nSR                  U5      $ )Nrf  z))r   
)r{   r  r  r   rq   rh  r   ri  )r;  line0endlr	  rj  s        rx   r  MutableBox.__str__  s    dii,,Dz**+1T$))_-E-E,FaHEDIINNEDz**+1-EIIED 3u:

 yyrw   ro   r   r(  r,  r.  r0  r2  r   r3  r5  r  r6  r7  r8  r:  r;  r=  r>  r?  r@  rA  r%  rD  rF  rG  r~  r  r  r'  r  r  r!  )/rq   rr   rs   rt   r  ru   r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  rN  r  r  r  r  r  r  r  r  r  r  r  r  rO  r[  r  rM  r   r   r  r   r   r  r  rv   ro   rw   rx   r  r  -  s5    L2&'(&$7+(') 7<OO/3O	O
>? DIX+X<@X	X
+%%-..%$4..8#4*+3 + +&+$   " Hrw   r  c                  $    \ rS rSr\S 5       rSrg)rb   i  c                X    [        U [        5      (       a  U $ [        [        U 5      5      $ r   )r{   r   rb   r  r  s    rx   r  TensorBox.create  s%    d122KD)**rw   ro   N)rq   rr   rs   rt   rJ  r  rv   ro   rw   rx   rb   rb     s    + +rw   c                  T    \ rS rSrS rS rSS jrSS jrSS jrS r	SS jr
S	 rS
rg)r  i  c                    [        U R                  [        [        45      (       a5  U R                  R	                  5       [
        R                  R                  ;   $ gr  )r{   r  rV  r$  r  rV   r   graph_inputsrB  s    rx   r  StorageBox.is_input_buffer  s=    dii+!?@@99%%'177+?+???rw   c                    [        U R                  [        5      =(       a5    U R                  R                  5       [        R
                  R                  ;   $ r   )r{   r  r  r  rV   r   r	  rB  s    rx   r<  StorageBox.is_module_buffer  s9    tyy>3 :		""$(9(99	
rw   c           	        [        U R                  [        [        [        [
        [        45      (       a  U R                  R                  5       $ [        U R                  [        [        [        [        45      (       d   [        U R                  5      5       eU R                  R                  5       nU R                  R                  5       n[        S [        U R                  R!                  5       U R                  R#                  5       U R                  R%                  5       S9U R                  S9U l        [&        R(                  R+                  U R                  5      U R                  l        [&        R(                  R/                  U R                  5        U R0                  U R                  l        XR                  l        X R                  l        U R                  R,                  $ )Nrw  rx  )r{   r  r  r  rV  r$  r  r  r  rH  ru  r  r   rW  rS  r   r   r   r   rV   r   r  r   r  r0  r4  r2  )r;  r4  r2  s      rx   r  StorageBox.realize  sS   II	
 	
 99%%''$))iD$%GHH 	
$IIK
 	
H ii//1II++-	"!yy++-ii))+YY'')
 
	 00;			""499- LL		 +		'		yy~~rw   c                    [        U R                  [        [        45      (       a:  U R                  R	                  5       R
                  S:  a  U R                  5         ggg)z<
Called on buffers we expect to be forced to realize later.
r1   N)r{   r  r  rH  r  nontrivial_read_countr  rB  s    rx   r  StorageBox.realize_hint  sI    
 tyy9i"899		**,BBQFLLN G :rw   c                    [        U R                  [        5      =(       a8    U R                  5       [        R
                  :  =(       d    U R                  5       $ r   )r{   r  r  r  r2   realize_acc_reads_thresholdr  rB  s    rx   r  !StorageBox.has_exceeded_max_reads  s@    $))Y/ 
NNvAAA )&&(	
rw   c                r  ^ US:  a  [        U R                  [        [        45      (       a  [	        U R                  5      (       a9  U R                  R                  5       mSS/n[        U4S jU 5       5      (       a  gU R                  5       [        R                  :  =(       d    U R                  5       $ g)zR
A heuristic to decide if we should realize a tensor
that is used multiple times.
r1   expsigmoidc              3  @   >#    U  H  oTR                   ;   v   M     g 7fr   )used_ops)r  r   opcounts     rx   r  5StorageBox.should_realize_on_reuse.<locals>.<genexpr>  s     @iG,,,is   TF)r{   r  r  rH  r   r  r  r  r2   realize_reads_thresholdr  )r;  r  	heavy_opsr  s      @rx   should_realize_on_reuse"StorageBox.should_realize_on_reuse  s    
 19DII	9/EFFdii  ))446"I.	@i@@@ 6#A#AA -**, rw   c                R    U R                  U5      (       a  U R                  5         g g r   )r  r  r  s     rx   r  StorageBox.mark_reuse  s!    ''..LLN /rw   c                6    U R                   R                  5       $ r   r  rB  s    rx   r  StorageBox.num_reads  r:  rw   r  Nr%  r  r   r5  )rq   rr   rs   rt   r  r<  r  r  r  r  r  r  rv   ro   rw   rx   r  r    s+    

B
$%rw   r  c                  8    \ rS rSr% S\S'   S\S'   SrS\S'   S	rg)
Subgraphi  r   r   ztorch.fx.GraphModulegraph_moduleNzOptional[GraphLowering]r   ro   )rq   rr   rs   rt   ru   r   rv   ro   rw   rx   r  r    s    
I&&%)E")rw   r  c                    U  Vs/ s H*  n[        U[        5      (       a  UR                  5       OUPM,     n n[        [	        S U  5       5      5      [        U 5      :  $ s  snf )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr   )r	  )r  r  s     rx   r  '_has_aliased_buffers.<locals>.<genexpr>+  s     ;7"V**7rL  )r{   r$  r  r   r,   )buffersr  s     rx   _has_aliased_buffersr  %  sd     F !+6? C CO  
 z;7;;<s7|KKs   1Ac                     ^  \ rS rSr% SrS\S'   SrS\S'   SrS\S'           SU 4S	 jjr\	SS
 j5       r
SS jrSrU =r$ )InvokeSubgraphi.  NOptional[Subgraph]subgraphzOptional[list[TensorBox]]operandsOptional[list[MultiOutput]]r  c                   > [         TU ]  S UUS9  Xl        [        R                  R                  U 5      U l        [        R                  R                  U 5        g r  )r  rC  r  rV   r   r  r   r  )r;  r  r  r   r  s       rx   rC  InvokeSubgraph.__init__4  sO     	 	 	

 !GG++D1		""4(rw   c                >  ^ [         R                  R                  R                  S   nU Vs/ s H  oDR                  S   PM     nnU Vs/ s H  o@R                  U5      PM     nnS n/ n[        U5       He  u  p[        U	[        5      (       a  UR                  U	5        M-  U" XX   R                  5       5      n
UR                  U R                  X5      5        Mg     UnUR                  cz  [         R                  R                  UR                  UUR                  S9Ul        [         R                  " UR                  5         UR                  R                   " U6   S S S 5        UR                  R"                  nS nU H*  n	[        U	[        5      (       a  M  U	R%                  5       n  O   Uc   e['        UU[)        US9S9mSU4S jjn[        U5       VVs/ s H  u  pU" X5      PM     nnnUTl        U$ s  snf s  snf ! , (       d  f       N= fs  snnf )	Nr  r  c                    U  Vs/ s H:  n[        U[        R                  5      (       a  UR                  R                  OUPM<     sn$ s  snf r   )r{   r   r  r   rl  )r   r   s     rx   handle_sym_expr.InvokeSubgraph.create.<locals>.handle_sym_exprK  s7    OUVv!:a#>#>AFFKKAEvVVVs   AA	gmexample_inputssubgraph_namer  )r  r  r   c           
       > [        U [        [        45      (       a  U $ [        [	        U R                  5       U R                  5       U R                  5       U R                  5       U R                  5       R                  S9T[        U4/5      $ )Nr  )r{   r   re  r  r"  r   r   r   r  r   r#  r|   )r
  indinvoke_subgraphs     rx   create_output,InvokeSubgraph.create.<locals>.create_outputs  s    &#8:N"OPP"%002$..0#__.%002%00299 $C[M
 
rw   )r
  rc   r  r   )rV   r   r
  r   rs  ry  r   r{   r   r  r   r  make_subgraphr  r   set_graph_handlerrungraph_outputsr   r  rg  r  )r  r  r  fx_operandsr   fake_operandsr  new_operandsr   operandexample_strider  r   r  r   r
  r  s                   @rx   r  InvokeSubgraph.create@  s    gg**//30;<1<
 3;;(Q%%a((;	W %h/LC'#899##G,!01C1J1J1L!M##C$=$=g$VW 0  >>!WW22((,&mm 3 HN
 $$X^^4""M2 5 .... Gg'<== ++-   !!!($F3
	  >Gw=OP=O	=+=OP")C =
 <, 54H Qs   G>HHH
Hc                &    UR                  U 5        g r   )codegen_invoke_subgraphr  s     rx   r  InvokeSubgraph.codegen  r	  rw   )r   r  )r  r  r  zlist[TensorBox]r   rg  r   r   )r  r  r  )rq   rr   rs   rt   r  ru   r  r  rC  r  r  r  rv   r  r  s   @rx   r  r  .  sj    #'H '*.H'.+/G(/
) 
),;
)EV
)	
) D DL. .rw   r  c                     ^  \ rS rSr% SrS\S'   SrS\S'   SrS\S'   SrS\S	'   Sr	S
\S'                 SU 4S jjr
\        SS j5       rSS jrSS jrSrU =r$ )Conditionali  Nr.  	predicate7Optional[list[Union[TensorBox, ShapeAsConstantBuffer]]]r  r  true_subgraphfalse_subgraphr  r  c                  > Xl         X l        X0l        X@l        [	        U/U-   5      u  px[
        T	U ]  S UUUS9  Ub  X`l        [        R                  R                  U 5      U l        [        R                  R                  U 5        g N)r   r   r  r  )r  r  r  r  _split_by_sym_typer  rC  r  rV   r   r  r   r  )
r;  r  r  r  r  r   r  sym_argsr	  r  s
            rx   rC  Conditional.__init__  s     # *, 2I;3I J"	 	 	
 (%6"GG++D1		""4(rw   c                    U R                  U5      nU Vs/ s H  oPR                  U5      PM     nn[        R                  R                  R                  S   nU Vs/ s H  oUR
                  S   PM     nnX#4 H  nUR                  b  M  [        R                  R                  UR                  UUR                  S9Ul        [        R                  " UR                  5         UR                  R                  " U6   S S S 5        M     UR                  R                  n	UR                  R                  n
SU	4SU
44 H&  u  p[        U	5      (       d  M  [        SU SU 35      e   [        U	5      [        U
5      :X  d   X45       e[        [!        X5      5       H  u  nu  pUR#                  5       UR#                  5       :X  d	   XU45       eUR%                  5       UR%                  5       :X  d	   XU45       eUR'                  5       R(                  UR'                  5       R(                  :X  a  M   XU45       e   [+        S U/U-    5       5      n[-        [        R                  R.                  R0                  [        R                  R                  R
                  R3                  S	S 5      5      nUc   S
5       e[5        UUUU[7        US9US9nSS jn[        [!        U	[        R                  R                  R
                  S   5      5       VVVVs/ s H  u  nu  nn[9        [;        UR#                  5       UR%                  5       UR=                  5        Vs/ s H  nU" U5      PM     snUR?                  5        Vs/ s H  nU" U5      PM     snUR'                  5       R(                  S9U[@        U4/5      PM     nnnnnUUl!        U$ s  snf s  snf ! , (       d  f       GMq  = fs  snf s  snf s  snnnnf )Nr  r  r  true_fnfalse_fnzVOutput aliasing is currently not supported in compiled torch.cond. The outputs of the z% subgraph of torch.cond are aliased: c              3  p   #    U  H,  n[        U[        5      (       a  M  UR                  5       v   M.     g 7fr   )r{   r   r   )r  os     rx   r  %Conditional.create.<locals>.<genexpr>  s,      
+a!67 ALLNN+s   66r  zcannot determine devicer  )r  r  r  r  r   r  c                \    [        U [        5      (       a  U $ U R                  R                  $ r   )r{   r   r   rl  )r   s    rx   _maybe_expr'Conditional.create.<locals>._maybe_expr  s"    !S!!66;;rw   r  )r   zUnion[int, torch.SymInt]r   zUnion[int, sympy.expr])"ry  rV   r   r
  r   rs  r  r  r   r  r  r  r  r{  r   r   r   r   r   r   r#  r  r(   r   r   ry  r  rg  r  r"  r   r   r|   r  )r  r  r  r  r  r   r  r  r  true_outputsfalse_outputsr   r  r   tofor   r  conditionalr  r
  merged_outputr  s                          rx   r  Conditional.create  s    %%i0	2:;(Q%%a((;gg**//30;<1< +H~~%!"!6!6,,#0"*-- "7 "
 ((8NN&&6 98 , }}22 44(,7*m9TUMD#L11$**./TU\T]_  V < C$66U8UU6$S%EFKAx==?bmmo5Br{B5<<>R\\^3@aR[@3==?))R]]_-C-CCPaR[PC G
  
[8+
 

 6GG&&GG  %%))*=tD
 !<#<<!!!#$F3/
	& /8L!''"6"6";";E"BC/
/**FM !,,. **,4A4F4F4HI4Hb+b/4HI6C6J6J6LM6LKO6LM!,,.55 
/ 	 
& &Y << 98b JM
s<   OO$OAO8
O..O8
O3/O8

O+	.
O8
c           	         UR                  U 5        UR                  U R                  5       U R                  [	        U S0 5      5        g r
  )codegen_conditionalr
  r  r  r   r  s     rx   r  Conditional.codegen  s9    ##D)88MMOT\\749Lb+Q	
rw   c                    [        U SS 5      =n(       aC  [        [        R                  R                  R
                  U5      nUc   eUR                  5       $ [        5       $ r
  r
  r
  s      rx   rt  $Conditional.get_unbacked_symbol_defs  r
  rw   )r  r   r  r  r  r  )r  rc   r  -list[Union[TensorBox, ShapeAsConstantBuffer]]r  r  r  r  r   rg  r  z,Optional[dict[sympy.Symbol, pytree.KeyPath]]r   r   )r  rb   r  r  r  r  r  r  r  r~  )rq   rr   rs   rt   r  ru   r  r  r  r  rC  r  r  r  rt  rv   r  r  s   @rx   r  r    s    "&I&HLHEL(,M%,)-N&-+/G(/)) @)  	)
 !) ") H) 
)8 TT T 	T
 @T Tl
   rw   r  c                    / n/ nU  HF  n[        U[        5      (       a  UR                  UR                  5        M5  UR                  U5        MH     X!4$ r   )r{   r   r  rl  )r   non_sym_argsr  r  s       rx   r  r    sO     LHc011OOCHH%$	  !!rw   c                     ^  \ rS rSr% SrS\S'   SrS\S'   SrS\S'   SrS\S'   Sr	S	\S
'               SU 4S jjr
\        SS j5       rSS jrSrU =r$ )	WhileLoopi%  Nr  carried_inputsadditional_inputsr  cond_subgraphbody_subgraphr  r  c                   > Xl         X l        X0l        X@l        [	        X-   5      u  pg[
        TU ]  S UUUS9  [        R                  R                  U 5      U l
        [        R                  R                  U 5        g r  )r  r  r  r  r  r  rC  rV   r   r  r   r  )	r;  r  r  r  r  r   r  r	  r  s	           rx   rC  WhileLoop.__init__-  su     -!2** 2>3U V"	 	 	
 GG++D1		""4(rw   c                \	   U Vs/ s H  oPR                  U5      PM     nnU Vs/ s H  oPR                  U5      PM     nnX4-   n[        R                  R                  R                  S   [        R                  R                  R                  S   -   nU Vs/ s H  oUR
                  S   PM     nnX4 H  n	U	R                  b  M  [        R                  R                  U	R                  UU	R                  S9U	l        [        R                  " U	R                  5         U	R                  R                  " U6   S S S 5        M     UR                  R                  n
UR                  R                  n[        U5      (       a  [        SU 35      e[        U
5      S:X  d   U
5       eU
S   n[        U[         5      (       dM  UR#                  5       [$        R&                  :X  d   U5       e[        UR)                  5       5      S:X  d   U5       e[        U5      S:  d   S5       eUS   R+                  5       n[        U5      [        U5      :X  d   X;45       e[-        [/        X;5      5       H  u  nu  nn      SS	 jnU" UR)                  5       UR)                  5       5        U" UR1                  5       UR1                  5       5        UR+                  5       UR+                  5       :X  d
   XUU45       eUR#                  5       UR#                  5       :X  d	   XU45       eUR3                  5       R4                  UR3                  5       R4                  :X  a  M   XU45       e   [7        UUUU[9        US
9S9n[-        U5       VVs/ s Hu  u  nn[;        [=        UR+                  5       UR#                  5       UR)                  5       UR1                  5       UR3                  5       R4                  S9U[>        U4/5      PMw     nnn[/        UU5       Hk  u  nnURA                  5       [        R                  RB                  ;   d  M4  [        R                  RD                  RG                  URA                  5       5        Mm     UUl$        U$ s  snf s  snf s  snf ! , (       d  f       GM  = fs  snnf )Nr  r  r  zOutput aliasing is currently not supported in compiled torch.while_loop. The outputs of the body_fn subgraph of torch.while_loop are aliased: r1   r   z9torch.while_loop is assumed to have at least one operand.c                ~    [        X5       H.  u  p#[        R                  R                  R	                  X#5        M0     g r   )r   rV   r   r   r  )	lhs_exprs	rhs_exprslhsrhss       rx   _guard_list_equals,WhileLoop.create.<locals>._guard_list_equalsy  s-     !$I 9HCGG$$11#; !:rw   r  )r  r  r  r  r   r  )r  list[Union[int, sympy.expr]]r  r  r   r   )%ry  rV   r   r
  r   rs  r  r  r   r  r  r  r  r{  r   r{   r   r   r   rm   r   r   r   r   r  r   r#  r  rg  r  r"  r|   r  r  r#
  r  r  )r  cond_fnbody_fnr  r  r   
all_inputsfx_all_inputsfake_all_inputsr  cond_outputsbody_outputsr  r   r   opbor  
while_loopr
  r  r  r   s                          rx   r  WhileLoop.createE  s    9GG1++A.G;LM;La..q1;LM#7
,,11"58L8L8Q8QRT8UU2?@-Q66%=-@ *H~~%!"!6!6,,#0"*-- "7 "
 ((8NN&&8 98 + }}22}}22-- XXdWeg  < A%3|3%O!233;;=EJJ.11.qzz|$),1,):" 	
G	
" A))+ >"c,&77W.9WW7$S%FGKAxB<7<7< < r{{}bkkm<r}}@ ==?bmmo5Jr67JJ5<<>R\\^3@aR[@3==?))R]]_-C-CCPaR[PC H" )/!!$F3

* '|4
 5	6 !,,. **,*!,,.!,,.55 
 5 	 
 NG4HC||~!5!55 ++//? 5 %
{ HM A 98j
s#   RRRRA<R(
R%	c                &    UR                  U 5        g r   )codegen_while_loopr  s     rx   r  WhileLoop.codegen  s    ""4(rw   )r  r  r  r  r   )r  r  r  r  r  r  r  r  r   rg  r   r   )r   r  r  r  r  r  r  r  r  )rq   rr   rs   rt   r  ru   r  r  r  r  rC  r  r  r  rv   r  r  s   @rx   r  r  %  s    NRNKRQUNU(,M%,(,M%,+/G(/)E) I)  	)
  ) ") 
)0 dd d F	d
 Id dL) )rw   r  c                  V   ^  \ rS rSr SSS. S	U 4S jjjjrS
U 4S jjrSS jrSrU =r$ )r   i  Nr
  c          
     t  > [         TU ]  UUUUUS US9  SSKJn  U V	s/ s H&  n	[	        U	[
        5      (       a  U	R                  OU	PM(     n
n	U" U/ UQU
Q7U5      nUc   eXl        [        R                  R                  R                  US 5      U l        U [        R                  R                  U'   g s  sn	f )N)r   r  r   )get_effect_key)r  rC  torch._higher_order_ops.effectsr  r{   r{  r   effect_typerV   r   effectful_opsry  prev_effect_buffer)r;  r   r  r	  r
  r  r   r  r  r'  uncovered_argsr  r  s               rx   rC  EffectfulKernel.__init__  s     	/ 	 	
 	C GR
FQz!_55AGG1<k 	 
 %V-O~-O-OQWX&&&&"#''"7"7";";K"N-1k*
s   -B5c                   > [         TU ]  5       nU R                  bG  UR                  R	                  [
        R                  " U R                  R                  5       5      5        U$ r   )r  r  r  r  r  r3   r  r  )r;  r  r  s     rx   r  EffectfulKernel.get_read_writes  sU    g-/"".!!$$T%<%<%E%E%GH rw   c                    gr  ro   rB  s    rx   r
   EffectfulKernel.has_side_effects  r]  rw   )r  r  r   r  r>  r   )	rq   rr   rs   rt   rC  r  r
  rv   r  r  s   @rx   r   r     s6     2 2 
2 2@ rw   r   c                      \ rS rSrSrg)r&	  i  ro   Nr  ro   rw   rx   r&	  r&	    s    rw   r&	  c                  h    \ rS rSr% SSKJr  S\S'   S\S'   S rSSS
 jjrSS jr	SS jr
SS jrSrg	)r{  i  r   )FakeScriptObjectr   r   +Union[FakeScriptObject, torch.ScriptObject]r   c                    U R                   $ r   rG  rB  s    rx   r  TorchBindObject.get_name  r  rw   Nc                    U R                   $ r   rG  r  s     rx   r  !TorchBindObject.codegen_reference  r  rw   c                    U R                   $ r   r   rB  s    rx   	get_valueTorchBindObject.get_value  rr  rw   c                    [        U R                  [        R                  5      (       a  U R                  $ U R                  R                  $ r   )r{   r   r   ScriptObjectreal_objrB  s    rx   r	  TorchBindObject.get_real_obj  s3    djj%"4"455::::&&&rw   c                f   U R                  5       n[        UR                  5       5      n[        R                  " U5      S   nU Vs/ s HE  n[        U[        R                  5      (       d  M$  UR                  5       UR                  5       -  PMG     nn[        R                  " S US5      $ s  snf )Nr   c                
    X-   $ r   ro   )r   ys     rx   rz  /TorchBindObject.get_buf_bytes.<locals>.<lambda>  s    QUrw   )r	  r~   __obj_flatten__r  r	  r{   r   r	  r	  numelr  r  )r;  real_script_obj	flat_dict
flat_elemsr   
flat_sizess         rx   get_buf_bytesTorchBindObject.get_buf_bytes  s    ++-88:;	((3A6
  
!U\\* )ANNqwwy( 	 

  2JBB
s   #B..%B.ro   r   r'  )r   r  )r   ztorch.ScriptObjectr@  )rq   rr   rs   rt   "torch._library.fake_class_registryr  ru   r  r  r$  r	  r4  rv   ro   rw   rx   r{  r{    s*    C
I66'
Crw   r{  c                  >    \ rS rSr% S\S'   S\S'   S rS
SS jjrS	rg)r	  i  r   r   r+  r   c                    U R                   $ r   rG  rB  s    rx   r  GeneratorState.get_name  r  rw   Nc                    U R                   $ r   rG  r  s     rx   r   GeneratorState.codegen_reference  r  rw   ro   r   r'  )rq   rr   rs   rt   ru   r  r  rv   ro   rw   rx   r	  r	    s    
I rw   r	  c                  j    \ rS rSrS	S jrS	S jrS
SS jjr\    SS j5       r\  SS j5       r	Sr
g)_CollectiveKerneli  c                    gr  ro   rB  s    rx   rJ  !_CollectiveKernel.should_allocate  r  rw   c                    gr  ro   rB  s    rx   r
  "_CollectiveKernel.has_side_effects  r]  rw   Nc                Z   [        U R                  5      [        R                  R                  L d   S5       eU R                  nUR
                  R                  U l        UR
                  R                   Vs/ s H!  o3R                  (       d  M  UR                  PM#     snU l
        g s  snf )Nz,Setting cpp kernel needs a valid op_overload)r   r  r   r  r  r  r   r  r  r  r  )r;  r  r  r   s       rx   r  %_CollectiveKernel.set_cpp_kernel_name  s    D$$%)>)>> 	
:	
> !!%~~22 #NN44.
4qFAFF4.
* .
s   7B(B(c                <   [         R                  R                     U R                  " X/UQ70 UD6u  nnnnn	S S S 5        W	(       a   U SU	 35       eW H  n
U
R	                  5         M     US   R                  5       nU " [        US9UUWW5      n[        R                  " U5      nUR                  R                  U Vs/ s H  n[        [        US9X5      PM     sn5        UR                  R                  U Vs/ s H  oR                  5       PM     sn5        SU;   a]  UR                  R                  [        [        US9US   U5      5        UR                  R                  US   R                  5       5        g g ! , (       d  f       GNQ= fs  snf s  snf )Nrt  r   r  r   )rV   r   r
	  r	  r  r   r  r  tree_leavesr  r  r  r
  r  r  )r  r  r  r   r   _example_outputr	  r	  r  r  
tensor_argr   r  inpsr  r  s                   rx   create_inplace _CollectiveKernel.create_inplace,  s    WW ""6CDCFC!  %E2C1D&EE$%J  & Q**,f%
 !!&)&&OSTt^Jf5sCtT	

 	!!T"BTc<<>T"BCF?##**z8&-P %%fUm&<&<&>? 9 . U #Cs   FF>F
Fc           
     f   [         R                  R                     U R                  " X/UQ70 UD6u  nnnnn	S S S 5        W	(       a   U SU	 35       eW H  n
U
R	                  5         M     [        W[        5      (       av  U R                  Xe5      nU " [        US9UUWW5      n[        U5       VVs/ s H(  u  p[        U R                  U5      U[        U4/5      PM*     snnUl        UR                  $ U " U R                  U5      UUWW5      nU/Ul        U$ ! , (       d  f       N= fs  snnf )Nr  r  )rV   r   r
	  r	  r  r{   r|   r
  rg  r   r  r
  r  )r  r  r  r   r   r	  r	  r	  r  r  rG  r   r  r   r%  s                  rx   create_out_of_place%_CollectiveKernel.create_out_of_placei  sK    WW ""6CDCFC!  %F3D2E&FF$%J  & nd++__[AF!0F "+>!: ";IA ((0AYK
 ";FN >>!$$^4F %XFNMO *s   D3/D-
D*)r  r  r   r   rw	  )r  !Union[TensorBox, list[TensorBox]]r   r   )r  rN  )rq   rr   rs   rt   rJ  r
  r  r  rI  rL  rv   ro   rw   rx   r=  r=    sV    
	
" $@>$@	$@ $@x *>* *rw   r=  c                  F   ^  \ rS rSrS r\SS j5       rSU 4S jjrSrU =r	$ )_WaitKerneli  c                &   U R                   S   n[        U[        5      (       a  UR                   S   /$ [        U[        5      (       aG  UR                   S   n[        U[        5      (       a!  UR                  S   u  p4UR                   U   /$ / $ / $ r  )r  r{   r=  r  r  )r;  r  collr   r   s        rx   get_volatile_reads_WaitKernel.get_volatile_reads  s    kk!nc,--JJqM?"[)) ::a=D$ 122QC())I Irw   c                v   [         R                  R                     U R                  X5      u  nnnnnS S S 5        W(       a   U SU 35       eU " [	        UR                  5       S9UWWW5      nUR                  R                  [        [	        UR                  5       S9X(5      5        g ! , (       d  f       N}= f)Nrt  r  )	rV   r   r
	  r	  r  r   r  r  r  )	r  r  r  rF  r	  r	  r  r  r  s	            rx   create_wait_WaitKernel.create_wait  s    WW ""6/!  %E2C1D&EE$cnn./
 	&&:S^^-=>L	
! s   B**
B8c                   > [         TU ]  5       nU R                  5       nU H@  nUR                  R	                  [
        R                  " UR                  5       5      5        MB     U$ r   )r  r  rS  r  r  r3   r  r  )r;  r  volatile_readsvrr  s       rx   r  _WaitKernel.get_read_writes  sS    g-/002 B!!,"6"6r{{}"EF !rw   ro   )r  rb   r   r   r>  )
rq   rr   rs   rt   rS  r  rV  r  rv   r  r  s   @rx   rP  rP    s&    * 
 
* rw   rP  c                V   [        U [        [        45      (       a  [        U 5      $ [        U [        [
        45      (       a5  [        [        R                     " 5       nU  H  nU[        U5      -  nM     U$ [        U [        R                  5      (       a  [        U 5      $ [        5       $ r   )r{   r*   r   r&   r}   r|   r,   r   r   rp	  r   r	  )r   r  r   s      rx   rp	  rp	    s    !h%&&$Q''	At}	%	%u||$&A,Q//A 	Au||	$	$$Q''|rw   )r   r   r   r   )r   r   r   r-  )r   Sequence[int]r   z&Callable[[Sequence[_T]], Sequence[_T]])r   z&Callable[[Sequence[_U]], Sequence[_V]]r   z&Callable[[Sequence[_T]], Sequence[_U]]r   z&Callable[[Sequence[_T]], Sequence[_V]]r   )r   z(Sequence[Union[int, torch.SymInt, Expr]]r   zOptional[ShapeEnv]r   r]  )r   Sequence[Union[int, Integer]]r   r]  r  )r   zLiteral[None]r   rm   r   r   )r   rc   r   rm   r   r  )r   r.  r   rm   r   zOptional[torch.Tensor])r   zOptional[Sequence[_T]]r   z Optional[Sequence[Optional[_T]]])r   z2Union[IRNode, OutputSpec, torch.device, None, str]r   r&  )r   z&Union[IRNode, torch.device, None, str]r   rm   )r   zUnion[Buffer, TensorBox]r  r   r   rm   )r  r1  r  r1  r  r1  r   rm   )r%  Union[TensorBox, BaseView]r&  z"Sequence[Union[int, torch.SymInt]]r   r_  )r   zUnion[Expr, Sequence[Expr]]r   r  r   rU   )r3  r   r   r  r2  rm   r   r2  )r   rc   r   rm   )TFNFN)r   rc   r  rm   r  rm   r  'Optional[Sequence[Union[int, Integer]]]r  rm   r  r`  r   ztuple[StorageBox, Layout])r   rc   r  r^  r   rm   )r   r1  r  r1  r   rm   )r   r  r   r   )r  r  r   rm   )r   z	list[Any]r   z-tuple[list[ShapeAsConstantBuffer], list[Any]])r   r   r   rC  (  
__future__r   rK  rH  r  r  loggingtextwrapr2  r  collections.abcr   r   r   r   r   enumr	   r
   r   r   r   r   r   r   r   r   r   typing_extensionsr   r   r   unittest.mockr   r   r   r   r   torch._export.serde.schema_exportserder
  r
  torch._library.utilsr
  r  r
  torch._loggingr   torch.fxtorch.utils._pytree_pytreer  torch._dynamo.utilsr   torch._export.serde.serializer   *torch._higher_order_ops.auto_functionalizer   torch._inductorr   torch._prims_commonr   r    r!   r"   r#   torch._subclasses.fake_tensorr$   %torch.fx.experimental.symbolic_shapesr%   r&   r'   r(   r)   r*   torch.utils._ordered_setr,   torch.utils._sympy.functionsr-   r.   r/   torch.utils._sympy.symbolr0   r_  r2   r3   codegen.commonr4   r5   r6   r7   r8   r9   r:   r;   	loop_bodyr<   ops_handlerr=   r>   r?   r@   runtime.benchmarkingrA   runtime.hintsrB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   virtualizedrT   rU   rV   torch.fx.noderW   codegen.cuda.cuda_templaterX   r   rY   rZ   r   ru   r   __version__r  r  ImportErrorr[   r\   r]   r   r^   r  r_   	getLoggerrq   r  rh  r  r~   r   rd   r
  rg   r   r   r   r   r   r  r  r   r   r   r   r   r   r   r   r  r  r+  rc   rO  r  r  r  r  r  rF  rH  INNER_FN_TYr8  rH  rR  ru  r  r  r   r  r!  r  r  r   r`  ro  r~  r
  r$  r  r  r  r(  r3  r?  rC  r   r  r"  r   r  r  r  r  r  r  rM  rV  rZ  r  re  r   r  r  r  rm   r|   PrimitiveInfoTyper  r3  r8  r[  rc  r  r  r  r  ry	  r	  r  r  r	  r	  r	  r 
  r
  r!
  r'
  r1
  rG
  r[
  r   r   r
  r
  r  rg  r  r  rb   r  r  r  r  r  r  r  r   r&	  r{  r	  r=  rP  rp	  ro   rw   rx   <module>r     sK   "         9 9 :  
 
 
 = <   ' ' 2 2 , ,   $ $ ( ? M #  :  0 L L * " 
    N N - :    $ * ) "8$% %L)$''NJ T]T]T]CI&) &C,-) -!			8??4	8yy~~'T  k	sDk!12K8STU	i 	 d#  $$D44 , ! $  TX	1>P	 TX
	1
>P

 
 N 
 N 
 O 
 O .2&*8!%	>9	>	>;('0     
	.#G&#G/#G  #GLp, p,f UA A AH ~
F ~
 ~
B& 
 
 
@ 
i 
 
F |$y!y!u=)< 8  JN<N<N +<NBF<N<N~ o

 o

 o

d 7AB7S9 7St#1 #L[
+ [
| @
5 @
 @
H 	 	 	 Q5 Q Qh	 !<@=A999 9 :	9
 9 ;9 9x:	 \
v \
 \
~ K K K\ (( ( (V 79( 79 79t (  : U; U Up Ph P Pf % % %POA OAd 6  " K| K K$ S| S S'9	<7 7 HTZ HT HTV& $I7V I7XPf P2T $%{ $%N   .Q* Q*h UxV x xv U&fi & & & 
K 
[ 
& 6  " 
F 
 
 UE4_ E4 E4P
B
_ B
J1> 1h #udCeCeT<Q6R1SST -$ -$`"| "
D=. D=NM M$( (& U>? > >B h9 hV UH< H HV U2l 2 2j
/ 
(*" *"ZV .G.L G.Tll l^&)| &)T-, -`< 6B5 B"F- F,H)l H)V,)| ,)^!T !TH-L -46;< 6;r U  
`*& `*F U
. 
 
< 
  8
, 8
z N N Nb+
 +T% T%n U*v * *L UY.\ Y. Y.x UH , H  H V"
"2" UG) G) G)T,n ,^	6 	 Cl C CD \  B BJ2# 2p yr  NJs   b2 2
c ?c 