o
    Zh&V                    @  s  U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZmZ d dlmZmZ d dlmZ d dlmZ d dl	mZmZmZmZmZmZmZmZmZ d dlmZmZmZ d d	l m!Z! d dl"Z"d d
l"m#Z#m$Z$m%Z% d dl&m'  m(  m)Z* d dl+m,  m-Z. d dl/Z0d dl1Z0d dl2m-  m3Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z>m?Z?m@Z@mAZAmBZB d dlCmDZD d dlEmFZFmGZGmHZHmIZImJZJmKZK d dlLmMZM d dlNmOZOmPZPmQZQ d dlRmSZS ddlTmUZUmVZV ddlWmXZXmYZYmZZZ ddlVm[Z[m\Z\m]Z]m^Z^m_Z_ ddl`maZa ddlbmcZcmdZdmeZemfZf ddlgmhZh ddlimjZjmkZk ddl-mlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{ ddl|m}Z}m~Z~mZ er{d dlmZ dd lmZ dd!lmZ dd"l-mZ neZd#ed$< zd dlZejZd%ZW n ey   dZd&ZY nw ed'Zed(Zed)Zeee#f Zd#ed*< eeee#f Zd#ed+< eeZejejd,d-Ze0j}jZ	 eed.eed.f d/d0eeeeeed.f d.d/d0f   f Zd#ed1< ejd%d2G d3d4 d4Zdfd9d:Zdgd>d?ZdhdCdDZdhdEdFZdidLdMZg dNZg dOZ	djdkdTdUZdldWdXZ	djdkdYdZZedmdnd_d`Zedmdodbd`Z	%dmdpded`ZdqdidjZdrdmdnZdsdpdqZdsdrdsZdtdwdxZdud}d~ZdvddZG dd0 d0Zetd&d2G dd dZetG dd deZdwddZetG dd deZetG dd deZededededededdZded< 	%dmdxddZetG dd deZeee# ee# ge~f ZG dd deZG dd deZG dd deZetG dd deZetG dd deZetG dd deZdyddZdyddZ	%	&		&	dzd{ddZÐd|ddZetG dd deZetG ddÄ deŃZetG ddń deŃZetG ddǄ deŃZetG ddɄ deŃZetG dd˄ deɃZetG dd̈́ deŃZetG ddτ deŃZG ddф deʃZetG ddӄ deZetG ddՄ de΃ZetG ddׄ de΃ZАd}ddڄZѐd~dd܄ZG ddބ dރZetG dd deӃZG dd deԃZG dd deԃZG dd deԃZG dd deZG dd deՃZetG dd deӃZG dd deԃZetd&d2G dd deZetd&d2G dd deeZG dd de܃ZG dd deރZG dd deރZetG dd deZetG dd deZetd&d2G dd de݃ZG dd  d e݃ZG dd deZeeeeeeeeeeef  f ZG dd dZG dd deZG dd deZG d	d
 d
eZG dd deZetd&d2G dd de݃ZG dd deZG dd deZetd&d2G dd deZetd&d2G dd deZG dd deZG dd deZG dd de܃ZG dd deZG dd  d eZG d!d" d"eZG d#d$ d$eZG d%d& d&eZG d'd( d(eZG d)d* d*eZG d+d, d,eZG d-d. d.eZG d/d0 d0eZG d1d2 d2eZ G d3d4 d4eZetd&d2G d5d6 d6ZG d7d8 d8eZetd&d2G d9d: d:eZetG d;d< d<eӃZG d=d> d>eZejG d?d@ d@eZG dAd. d.eZG dBdC dCeZ	etd&d2G dDdE dEeZ
ddHdIZetd&d2G dJdK dKeZetd&d2G dLdM dMeZddQdRZetd&d2G dSdT dTeZG dUdV dVeZG dWdX dXeZetG dYdZ dZeZetG d[d\ d\eZG d]d^ d^eZG d_d` d`eZddddeZdS (      )annotationsN)	GeneratorIterableSequence)AbstractContextManagernullcontext)Enum)partial)	AnyCallableClassVarLiteralOptionaloverloadTYPE_CHECKINGTypeVarUnion)assert_neverNever	TypeAlias)patch)ExprIntegerSymbol)identity)GraphModuleSerializer)can_auto_functionalize)metrics)compute_required_storage_lengthis_boolean_dtypeis_float_dtypemake_channels_last_strides_for
StrideType)get_schema_info)compute_unbacked_bindingsfree_unbacked_symbolsrebind_unbackedresolve_unbacked_bindingsShapeEnvSymTypes
OrderedSet)CleanDivFloorDivModularIndexing)SymT   )configdependencies)BackendFeatureget_scheduling_for_deviceindex_prevent_reordering)Depextract_free_unbacked_symbols#extract_input_node_reduction_rangesextract_read_writesvar_builder)LoopBody)OpCounterCSEOpCountResultReductionType	StoreMode)benchmarker)DevicePropertiesReductionHint)argsortargsort_symcache_on_selfceildivconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warningget_kernel_metadatair_dataclass
is_dynamicis_gpu	sympy_dotsympy_index_symbolsympy_index_symbol_with_prefixsympy_product
sympy_subs)opsOpsValueV)Node)CUDATemplate)GraphLowering)IndentedBufferr   rW   TF_T_U_V_IntLike_NumLikez  prefix	TensorBoxr   IRNode_NodeOrNodes)frozenc                   @  s.   e Zd ZU ded< ded< ded< ded< d	S )
GraphPartitionSignaturez5dict[str, Union[IRNode, sympy.Expr, TorchBindObject]]input_nodeslist[IRNode]Zoutput_nodeszdict[str, bool]Zinput_deallocationboolZskip_cudagraphN__name__
__module____qualname____annotations__ rn   rn   A/var/www/auris/lib/python3.10/site-packages/torch/_inductor/ir.pyre      s
   
 re   node_or_nodesOptional[_NodeOrNodes]returnNonec                   s   d fdd  |  d S )Nnodesrq   rr   rs   c                   s   | d u rd S t | ttfr| D ]} | qd S t | tr*|  D ]} | q!d S t | ttttt	j
jjttttf	sEJ dt|  dd S )NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])
isinstancelisttupledictvalues
ExpandViewDynamicScalarAssertScalarra   sympylogicboolalgBooleanr   intEffectfulKernelShapeAsConstantBuffertype)rt   node_check_tensorboxrn   ro   r      s2   


z%validate_ir.<locals>._check_tensorbox)rt   rq   rr   rs   rn   )rp   rn   r   ro   validate_ir   s   r   namestrCallable[..., OpsValue]c                   s    t  tsJ d fdd}|S )	Nargsobjectkwargsrr   rT   c                    s   t t | i |S N)getattrrS   r   r   r   rn   ro   fn      zops_wrapper.<locals>.fn)r   r   r   r   rr   rT   )ru   r   )r   r   rn   r   ro   ops_wrapper   s   r   orderSequence[int]&Callable[[Sequence[_T]], Sequence[_T]]c                   s(   t t| tt|  d fdd}|S )NindexSequence[_T]rr   c                   0   t  t ks
J  fddtt  D S )Nc                      g | ]} |  qS rn   rn   .0i)r   	inv_orderrn   ro   
<listcomp>       z4inverse_reorder.<locals>.reindex.<locals>.<listcomp>lenranger   r   r   ro   reindex      z inverse_reorder.<locals>.reindexr   r   rr   r   )rx   zipr   r   r   r   rn   r   ro   inverse_reorder   s   r   c                   s   d fdd}|S )Nr   r   rr   c                   r   )Nc                   r   rn   rn   r   )r   r   rn   ro   r      r   z1same_reorder.<locals>.reindex.<locals>.<listcomp>r   r   r   r   ro   r      r   zsame_reorder.<locals>.reindexr   rn   r   rn   r   ro   same_reorder   s   r   reindex1&Callable[[Sequence[_U]], Sequence[_V]]reindex2&Callable[[Sequence[_T]], Sequence[_U]]&Callable[[Sequence[_T]], Sequence[_V]]c                   s   d fdd}|S )Nr   r   rr   Sequence[_V]c                       | S r   rn   r   r   r   rn   ro   r        z fuse_reindexing.<locals>.reindex)r   r   rr   r   rn   )r   r   r   rn   r   ro   fuse_reindexing  s   r   )   r      r0   )   r   r   r   r0   seq(Sequence[Union[int, torch.SymInt, Expr]]	shape_envOptional[ShapeEnv]c                 C  s"   |du r
t | }|S t|| }|S )z1
    Convert strides to fill order (argsort)
    N)rC   rD   )r   r   
sorted_idxrn   rn   ro   get_fill_order  s
   
r   Sequence[Union[int, Integer]]c                   s0   dd t | D   fddtt| D }|S )z
    Convert stride order to fill order
    For channel last format,

    stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0]
    c                 S     i | ]\}}||qS rn   rn   r   idxposrn   rn   ro   
<dictcomp>%      z+stride_order2fill_order.<locals>.<dictcomp>c                      g | ]} | qS rn   rn   r   lookuprn   ro   r   &      z+stride_order2fill_order.<locals>.<listcomp>)	enumerater   r   )r   
fill_orderrn   r   ro   stride_order2fill_order  s   r   c                 C  s>   t | |}dd tt| D }t|D ]\}}|||< q|S )z)
    Convert strides to stride order
    c                 S     g | ]}d qS r   rn   r   _rn   rn   ro   r   1      z$get_stride_order.<locals>.<listcomp>)r   r   r   r   )r   r   r   outr   elemrn   rn   ro   get_stride_order*  s
   

r   xLiteral[None]guard_shaperh   c                 C     d S r   rn   r   r   rn   rn   ro   ir_node_to_tensor7     r   torch.Tensorc                 C  r   r   rn   r   rn   rn   ro   r   ;  r   Optional[IRNode]Optional[torch.Tensor]c                   s   | d u rd S |st jjj nt  fdd|  D }t| r, fdd|  jD }nt	
|}|  }|  }t|}t|}t jjj  tj||||d }W d    |S 1 s_w   Y  |S )Nc                      g | ]} |qS rn   rn   r   sZshape_fnrn   ro   r   J  r   z%ir_node_to_tensor.<locals>.<listcomp>c                   r   rn   rn   r   r   rn   ro   r   M  r   )sizestridedtypedevice)rU   graphsizevars	size_hintr   get_sizeis_storage_and_layout
get_layoutr   FlexibleLayoutcontiguous_strides	get_dtype
get_devicerH   r   Zsuppress_guardstorchZempty_stridedZzero_)r   r   r   r   r   r   trn   r   ro   r   ?  s.   

valueOptional[Sequence[_T]] Optional[Sequence[Optional[_T]]]c                 C  s   t | tr
| s
d gS | S r   )ru   rv   r   rn   rn   ro   may_convert_to_optional[  s   r   2Union[IRNode, OutputSpec, torch.device, None, str]Optional[str]c                 C  sb   t | ts	| d u r| S t | tjr| jS t | ttfr!t|  S t	d|  dt| j
 d d S )Nzget_device_type(: ))ru   r   r   r   r   rb   
OutputSpecget_device_typer   r   rj   r   rn   rn   ro   r  e  s    r  &Union[IRNode, torch.device, None, str]c                 C  sl   t | }|dv rtt| ddkrdS dS |d u s"t| }d u r$dS ddlm} t|ts1J t||S )N)cpucudaZ_backendtritonTFr0   )TritonScheduling)	r  r   r1   r4   Zcodegen.tritonr  ru   r   
issubclass)r   r   Zdevice_schedulingr  rn   rn   ro   	is_tritonq  s   
r	  c                 C  s   t | dkS )Nr  )r  r  rn   rn   ro   is_cpu  r   r
  Union[Buffer, TensorBox]	alignmentr   c                   s~   t tr d u rdS t fddtt d D }tjj	
 d dkp:tjj	
 d dk}|o>|S )NFc                 3  s.    | ]}t jj |   d kV  qdS )r   N)rU   r   r   r   
get_strider   r  r   rn   ro   	<genexpr>  s
    
z-is_aligned_realized_tensor.<locals>.<genexpr>r0   )ru   rb   maybe_get_strideallr   r   r  rU   r   r   r   r   )r   r  Zaligned_stridesZaligned_last_dimrn   r  ro   is_aligned_realized_tensor  s   r  strides1Sequence[_IntLike]strides2shapec                 C  s   t |t | krt | t |ksJ t|| |D ]'\}}}tjj|dr&qtjj||s?tjj|tjj|ks? dS qdS )zP
    Returns true if the strides are equal, ignoring dimensions of size 1 .
    r0   FT)r   r   rU   r   r   statically_known_leqstatically_known_equalssymbolic_hint)r  r  r  dims1s2rn   rn   ro   significant_strides_equal  s   $r  tensorUnion[TensorBox, BaseView]strides"Sequence[Union[int, torch.SymInt]]c                 C  s   t | s| S tdd t||  D r| S t||  |  s"| S t| \}}g |j}t|  D ]\}}t	j
j|drE|| ||< q3t|j|j|j||j}tt||dS )a  
    Tries to match the strides of the tensor to those in the meta_strides. Strides of insignificant
    dimensions - size 0 or 1 - will be updated.

    If there are real stride differences (NHWC vs NCHW), or the tensor is not realized, then the input will be returned
    c                 s  $    | ]\}}t jj||V  qd S r   rU   r   r   r  r   r  r  rn   rn   ro   r    
    
z2try_match_insignificant_strides.<locals>.<genexpr>r0   datalayout)r   r  r   r  r  r   as_storage_and_layoutr   r   rU   r   r   r  FixedLayoutr   r   r   offsetra   ReinterpretView)r  r!  storage
old_layout
new_strider   r   
new_layoutrn   rn   ro   try_match_insignificant_strides  s,   

r2  c                   @  s  e Zd ZU e Zded< ejddZded< ejddZ	ded< ejddZ
d	ed
< eejdddZdddZdddZdddZdddZdddZdd!d"Zddd'd(Z	#ddd,d-Zdd/d0Zdd2d3Zdd5d6Zdd8d9Zdd;d<Zdd>d?Zdd@dAZddCdDZddFdGZe ddIdJZ!ddLdMZ"ddNdOZ#ddQdRZ$dddVdWZ%ddYdZZ&dd\d]Z'dd^d_Z(ddadbZ)ddddeZ*ddgdhZ+ddidjZ,ddkdlZ-ddmdnZ.dddqdrZ/ddudvZ0ddwdxZ1ddydzZ2dd{d|Z3	ddddZ4dddZ5dddZ6	ddddZ7dddZ8dddZ9dddZ:dddZ;dddZ<dddZ=dddZ>dddZ?dddZ@dddZAdddZBdddZCdddZDeErTe dddZFdSS dSS )rb   zClassVar[OrderedSet[Any]]_current_originsF)initOrderedSet[Any]originsOptional[list[str]]	tracebackOptional[torch.fx.Node]origin_nodeOrderedSet[Node]rr   Generator[None, None, None]c                 c  s.    t j}|| B t _z	d V  W |t _d S |t _w r   )rb   r3  )r6  oldrn   rn   ro   current_origins  s   
zIRNode.current_originsattrr   r   r
   rs   c                 C  s   t | || d S r   )r   __setattr__)selfr?  r   rn   rn   ro   _post_init_setattr  s   zIRNode._post_init_setattrc                 C  s<   |  dt| j |  dtjrt nd  |  dd  d S )Nr6  r8  r:  )rB  r+   r3  r1   Zdebug_ir_tracebackr8  format_stackrA  rn   rn   ro   __post_init__  s
   zIRNode.__post_init__OrderedSet[str]c                 C     t dd |  D S )Nc                 s      | ]}|j V  qd S r   r   r   deprn   rn   ro   r        z(IRNode.get_read_names.<locals>.<genexpr>r+   	get_readsrD  rn   rn   ro   get_read_names     zIRNode.get_read_namesc                 C     | j S r   )r8  rD  rn   rn   ro   get_traceback     zIRNode.get_tracebackc                 C  rP  r   r:  rD  rn   rn   ro   get_origin_node  rR  zIRNode.get_origin_nodeOptional[Operation]c                 C  r   r   rn   rD  rn   rn   ro   get_defining_op  r   zIRNode.get_defining_opTshortenrh   Sequence[str]c                 C  s:   dt | dd }|rt|dkr|d d  d}|gS )Nzorigins=r6   @   =   z...)r   r   )rA  rW  r6  rn   rn   ro   common_repr  s   zIRNode.common_reprlinesSequence[object]	multilinec                 C  sb   t |t | | }t tt|}|r&td|}t| j d| dS t| j d| dS )Nz,
z(
z
)(r   )rv   r\  mapr   indentjoinr   rj   )rA  r]  rW  r_  	new_linesrn   rn   ro   
str_helper  s   zIRNode.str_helpertorch.dtypec                 C  rP  r   r   rD  rn   rn   ro   r     rR  zIRNode.get_dtypeOptional[torch.dtype]c                 C      z|   W S  ty   Y d S w r   )r   NotImplementedErrorrD  rn   rn   ro   maybe_get_dtype  
   
zIRNode.maybe_get_dtypeLayoutc                 C     t dt|  d)Nz#get_layout() is not implemented by !rj  r   rD  rn   rn   ro   r     r   zIRNode.get_layoutOptional[Layout]c                 C  ri  r   )r   rj  rD  rn   rn   ro   maybe_get_layout  rl  zIRNode.maybe_get_layoutr   c                 C     |   S r   )r   rD  rn   rn   ro   get_output_spec%     zIRNode.get_output_specOptional[OutputSpec]c                 C  ri  r   )rt  rj  rD  rn   rn   ro   maybe_get_output_spec(  rl  zIRNode.maybe_get_output_specc                 C  s   t |  tS )z4True for single tensor output (excludes MultiOutput))ru   rw  rm  rD  rn   rn   ro   has_tensor_output.  s   zIRNode.has_tensor_outputSequence[Expr]c                 C  rn  )Nz!get_size() is not implemented by ro  rp  rD  rn   rn   ro   r   2  r   zIRNode.get_sizeOptional[Sequence[_IntLike]]c                 C  ri  r   )r   rj  rD  rn   rn   ro   maybe_get_size5  rl  zIRNode.maybe_get_size.Union[_IntLike, sympy.Rel, Sequence[_IntLike]]c                 C  rs  r   r   rD  rn   rn   ro   r  ;     zIRNode.shaper   c                 C     t |  S r   )rQ   r   rD  rn   rn   ro   	get_numel?  r   zIRNode.get_numelc                 C     t jjt|  dS Nr   rU   r   r   is_expr_static_and_truer}   Eqr  rD  rn   rn   ro   is_zero_elementsB     zIRNode.is_zero_elementsr   c                 C     t dt|  )a)  
        If the IRNode refers to data which has not been materialized (e.g.,
        it is a Pointwise/Reduction that could potentially have more
        compute fused into it), realize the IRNode into physical memory,
        ending the possibility of fusing into it, but allowing, e.g., multiple
        users to access the data without having to recompute.

        Check StorageBox.realize for a particularly notable implementation.

        TODO(ezyang): I think, in principle, every IRNode should have an
        implementation of this, and most of the time no-op is OK, but you
        really do have to audit each IRNode for this, so for now, raise
        an error if it's not implemented.  Note that some code in graph.py
        will catch this thrown error and suppress it with a warning.
        zrealize NYI on rp  rD  rn   rn   ro   realizeE  s   zIRNode.realizeNwriterOptional[IndentedBuffer]c                 C  r  )Nzcodegen_reference NYI on rp  rA  r  rn   rn   ro   codegen_referenceW     zIRNode.codegen_referenceOptional[torch.device]c                 C  r   r   rn   rD  rn   rn   ro   r   Z  r   zIRNode.get_devicetorch.devicec                 C  s   |   }|d us
J |S r   r   rA  r   rn   rn   ro   get_device_or_error]  s   zIRNode.get_device_or_errorc                 C     dS NFrn   rD  rn   rn   ro   has_exceeded_max_readsb  r   zIRNode.has_exceeded_max_reads$Callable[[Sequence[Expr]], OpsValue]c                 C     t t| jr   rj  r   rj   rD  rn   rn   ro   make_loadere     zIRNode.make_loader Callable[[Sequence[Expr]], Expr]c                 C  r  r   r  rD  rn   rn   ro   make_indexerh  r  zIRNode.make_indexerr  c                 C  r  r   r  rD  rn   rn   ro   r  k  r  zIRNode.get_stridec                 C  ri  r   )r  rj  rD  rn   rn   ro   r  n  rl  zIRNode.maybe_get_stridec                 C  r  r   r  rD  rn   rn   ro   get_namet  r  zIRNode.get_namec                 C  ri  r   )r  rj  rD  rn   rn   ro   maybe_get_namew  rl  zIRNode.maybe_get_name	thresholdOptional[int]c                 C  r  r  rn   rA  r  rn   rn   ro   has_large_inner_fn}  r   zIRNode.has_large_inner_fnusersr   c                 C  r   r   rn   rA  r  rn   rn   ro   
mark_reuse  r   zIRNode.mark_reusec                 C  r   r   rn   rD  rn   rn   ro   realize_hint  r   zIRNode.realize_hintc                 C  r  r   r  rD  rn   rn   ro   unwrap_view  r  zIRNode.unwrap_viewc                 C  r  r   r  rD  rn   rn   ro   freeze_layout  r  zIRNode.freeze_layoutr   	list[int]allow_paddingc                 C  r  r   r  rA  r   r  rn   rn   ro   freeze_layout_with_stride_order     z&IRNode.freeze_layout_with_stride_orderc                 C  r  r   r  rA  r   rn   rn   ro   freeze_layout_with_fill_order  r  z$IRNode.freeze_layout_with_fill_orderr   list[_IntLike]c                 C  r  r   r  rA  r   rn   rn   ro   freeze_layout_with_same_order  r  z$IRNode.freeze_layout_with_same_orderexact_stridesc                 C  r  r   r  rA  r  r  rn   rn   ro    freeze_layout_with_exact_strides  r  z'IRNode.freeze_layout_with_exact_stridesdependencies.ReadWritesc                 C  r  r   r  rD  rn   rn   ro   get_read_writes  r  zIRNode.get_read_writesOrderedSet[Dep]c                 C  
   |   jS r   r  readsrD  rn   rn   ro   rM       
zIRNode.get_readsc                 C  r  r   )r   rM  rD  rn   rn   ro   	num_reads  r   zIRNode.num_readsr]   c                 C  r  r   r  rD  rn   rn   ro   get_storage_numel  r  zIRNode.get_storage_numelOrderedSet[Symbol]c                 C  r  r   r  rD  rn   rn   ro   get_unbacked_symbol_uses  r  zIRNode.get_unbacked_symbol_usesc                 C  r  r   r  rD  rn   rn   ro   get_reduction_type  r  zIRNode.get_reduction_typeSequence[sympy.Expr]c                 C  r  r   r  rD  rn   rn   ro   get_reduction_size  r  zIRNode.get_reduction_sizec                 C  r  r  rn   rD  rn   rn   ro   	is_extern  r   zIRNode.is_externc                 C  r  r  rn   rD  rn   rn   ro   is_no_op  r   zIRNode.is_no_opr   c                 C  r  r   r  r  rn   rn   ro   constant_to_device  r  zIRNode.constant_to_devicec                 C  r  r   r  rD  rn   rn   ro   get_mutation_names  r  zIRNode.get_mutation_namesc                 C  r  r   r  rD  rn   rn   ro   get_operation_name  r  zIRNode.get_operation_namec                 C  r  r   r  rD  rn   rn   ro   get_inputs_that_alias_output  r  z#IRNode.get_inputs_that_alias_outputc                 C  r   r   rn   rD  rn   rn   ro   r     r   zIRNode.dtype)r6  r;  rr   r<  )r?  r   r   r
   rr   rs   rr   rs   rr   rF  )rr   r7  rr   r9  rr   rU  T)rW  rh   rr   rX  )TT)r]  r^  rW  rh   r_  rh   rr   r   rr   rf  )rr   rh  rr   rm  )rr   rq  rr   r   )rr   rv  rr   rh   rr   ry  )rr   rz  )rr   r|  rr   r   rr   r   r   r  r  rr   r   rr   r  rr   r  rr   r  rr   r  rr   r  rr   r   r  r  rr   rh   r  r   rr   rs   rr   rb   Fr   r  r  rh   rr   rs   r   r  rr   rs   r   r  rr   rs   r  r  r  rh   rr   rs   rr   r  rr   r  rr   r   rr   r]   rr   r  rr   r  r   r  rr   rb   rr   rX  )Grj   rk   rl   r+   r3  rm   dataclassesfieldr6  r8  r:  staticmethod
contextlibcontextmanagerr>  rB  rE  rN  rQ  rT  rV  r\  re  r   rk  r   rr  rt  rw  rx  r   r{  propertyr  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rM  r  r  r  r  r  r  r  r  r  r  r  r   r   rn   rn   rn   ro   rb     s   
 













































c                   @  s   e Zd Zd.ddZd/ddZd0d	d
Zd1ddZd2ddZd3ddZd3ddZ	d4ddZ
d5ddZd6ddZd7d d!Zd8d#d$Zd9d&d'Zd9d(d)Zd:d+d,Zd-S );	Operationrr   rs   c                 C  s
   d | _ d S r   Zoperation_namerD  rn   rn   ro   rE    r  zOperation.__post_init__r  c                 C     t r   rj  rD  rn   rn   ro   r     r   zOperation.get_devicer9  c                 C     t | dsJ | jS Nr:  )hasattrr:  rD  rn   rn   ro   rT       zOperation.get_origin_noder5  c                 C  r  )Nr6  )r  r6  rD  rn   rn   ro   get_origins  r  zOperation.get_originsr   c                 C  s   | j d usJ | j S r   r  rD  rn   rn   ro   r    r  zOperation.get_operation_namerh   c                 C  r  r  rn   rD  rn   rn   ro   r    r   zOperation.is_externc                 C  r  r  rn   rD  rn   rn   ro   r    r   zOperation.is_no_opr  c                 C  r  r   r  rD  rn   rn   ro   r    r   zOperation.get_read_writesr   c                 C  s   ||   v S r   )rN  )rA  r   rn   rn   ro   
is_user_of  r   zOperation.is_user_ofrF  c                 C  rG  )Nc                 s  rH  r   r   rI  rn   rn   ro   r    rK  z+Operation.get_read_names.<locals>.<genexpr>rL  rD  rn   rn   ro   rN    rO  zOperation.get_read_namesr  c                 C  r  r   r  rD  rn   rn   ro   rM    r  zOperation.get_readslist[Buffer]c                 C  r  r   r  rD  rn   rn   ro   get_outputs  r   zOperation.get_outputsOrderedSet[sympy.Symbol]c                 C     t  S r   r*   rD  rn   rn   ro   get_unbacked_symbol_defs  rR  z"Operation.get_unbacked_symbol_defsc                 C  r  )a  
        Returns the unbacked symbols which are required to be in scope in
        order to successfully perform codegen for this buffer.  For example,
        a buffer that corresponds to an extern kernel call that takes i0 as
        an argument would return {i0} here.  This is used to generate necessary
        dependencies that ensure we actually bind i0 in codegen before you
        try to use it.

        Note that this is NOT transitive; in particular, if this buffer takes
        in as input another buffer with dynamic shape (e.g., (i0,)), we will
        not report it here, because you will already have a dependency
        on that buffer, which will eventually have a dependency on i0 if
        necessary.
        r*   rD  rn   rn   ro   r    s   z"Operation.get_unbacked_symbol_usesr   c                 C  r  )z
        Gets extra global memory size needed by this buffer.
        Some algorithms (e.g. group gemm) may require extra global memory in the generated code.
        r   rn   rD  rn   rn   ro   get_workspace_size  s   zOperation.get_workspace_sizeNr  r  r  )rr   r5  r  r  r  )r   r   rr   rh   r  r  rr   r  rr   r   r  )rj   rk   rl   rE  r   rT  r  r  r  r  r  r  rN  rM  r  r  r  r  rn   rn   rn   ro   r    s     













r  c                      s$  e Zd ZU ded< ded< ded< ded< dNddZdOddZdP fddZdQddZeZdRddZ	dSddZ
dTddZdTd d!ZedUd&d'ZeejfdVd*d+ZedWd-d.ZdXd0d1ZedQd2d3ZdYdZd8d9ZdNd:d;Zd[d=d>Zd\d@dAZd]dCdDZd^dFdGZd_dIdJZd`dLdMZ  ZS )aLoopsr  r   rf  r   Callable[..., Any]inner_fnr  rangesrr   r  c                 C  s&   t  jg dd | jD |  R  S )Nc                 s      | ]}t |V  qd S r   r%   r   ern   rn   ro   r        z1Loops.get_unbacked_symbol_uses.<locals>.<genexpr>)r+   unionr	  inner_fn_free_unbacked_symbolsrD  rn   rn   ro   r    s
   zLoops.get_unbacked_symbol_usesnamesrX  r   c                   sF     d jj dt j  g fdd|D  d jg S )N'c                   s    g | ]}| d t  | qS =)r   )r   r   rD  rn   ro   r   "  s     z!Loops._to_str.<locals>.<listcomp>origin_node=)re  r   r   r   r   inner_fn_strr:  )rA  r  rn   rD  ro   _to_str  s   zLoops._to_strrs   c                   s   t    d S r   )superrE  rD  	__class__rn   ro   rE  &  r  zLoops.__post_init__c                 C  
   |  dS )Nr	  r  rD  rn   rn   ro   __str__)  r  zLoops.__str__r  c                 C  rP  r   r   rD  rn   rn   ro   r   .  rR  zLoops.get_devicer9  c                 C  rP  r   rS  rD  rn   rn   ro   rT  1  rR  zLoops.get_origin_nodery  c                 C  rP  r   r  rD  rn   rn   ro   r   4  rR  zLoops.get_sizec                 C  rP  r   r  rD  rn   rn   ro   get_pointwise_size7  rR  zLoops.get_pointwise_sizer   r
   r   ra   c                 O  sN   | dd }| dd }| |i |}|d| |d|p |j t|S )Nr:  r8  )poprB  r8  ra   create)clsr   r   r:  tbrrn   rn   ro   r"  :  s   
zLoops.creater`   r/   c                   s    fddt | D S )Nc                   s*   g | ]\}}|d krt jjnt |qS r0   )r}   SZerorP   )r   nr   r_   rn   ro   r   J      z Loops._index.<locals>.<listcomp>)r   )r	  r`   rn   r_   ro   _indexH  s   
zLoops._indexr=   c              	   C  s   t t }t|2 ttdd | j|    |	 W  d    W  d    S 1 s0w   Y  W d    d S 1 s@w   Y  d S Nallow_indexingT)
r<   rU   ZMockHandlerZset_ops_handlerr   r   r   r  inner_fn_argsgetvalue)rA  Z	opcounterrn   rn   ro   inner_fn_opcountO  s   RzLoops.inner_fn_opcountSequence[Sequence[_IntLike]]c                 C  s   |  | jfS r   )r+  r	  rD  rn   rn   ro   r.  Y  r  zLoops.inner_fn_argsc                 C  s   t jj| jg|  R  S r   )rU   ZKernelFormatterHandlerZir_to_stringr  r.  rD  rn   rn   ro   r  \  s
   zLoops.inner_fn_strNr  r  rh   c                 C  s&   |d u rd}t |tj}|  j|kS r  )maxr1   Zrealize_opcount_thresholdr0  Znum_opsr  rn   rn   ro   r  b  s   zLoops.has_large_inner_fnc                 C  s   |  | j}t| j|S r   )r+  r	  r7   r  )rA  r   rn   rn   ro   r  h  s   z$Loops.inner_fn_free_unbacked_symbolsr  c                 C  sv   t tdd* |  r t|  |  |  jW  d    S t|  |  jW  d    S 1 s4w   Y  d S r,  )	r   r   r   r  r9   r  r   r  r  rD  rn   rn   ro   rM  l  s   $zLoops.get_readsrF  c                 C     t |  jS r   )r+   r0  read_buffersrD  rn   rn   ro   rN  z  r  zLoops.get_read_namesr   c                 C  r3  r   )r   r0  r4  rD  rn   rn   ro   r  }  r  zLoops.num_readsr  c                 C  rn  )Nz+get_reduction_size() is not implemented by ro  rp  rD  rn   rn   ro   r       zLoops.get_reduction_sizer   c                 C  rn  )Nz+get_reduction_type() is not implemented by ro  rp  rD  rn   rn   ro   r    r5  zLoops.get_reduction_typerb   c                 C  rn  )Nz+constant_to_device() is not implemented by ro  rp  r  rn   rn   ro   r    r5  zLoops.constant_to_devicer  )r  rX  rr   r   r  r  r  r  r  )r   r
   r   r
   rr   ra   )r	  r  r`   r/   rr   ry  )rr   r=   rr   r1  r   r  r  r  r  r  r  r  ) rj   rk   rl   rm   r  r  rE  r  __repr__r   rT  r   r   classmethodr"  r  r/   INDEXr+  rE   r0  r.  r  r  r  rM  rN  r  r  r  r  __classcell__rn   rn   r  ro   r    s>   
 







	





r  r   Union[Expr, Sequence[Expr]]r   rf  rT   c                C  s"   |j rttd|S td|S )Nnanr   )is_floating_pointrS   constantfloat)r   r   rn   rn   ro   nop_loader_fn  s   r@  c                   @  s>   e Zd ZdddZdddZdd	d
ZdddZdddZdS )	Pointwiserr   r  c                 C  s   |   rtt| jdS | jS Nrg  )r  r	   r@  r   r  rD  rn   rn   ro   r    s   zPointwise.make_loaderr  c                 C  s   g S r   rn   rD  rn   rn   ro   r    r   zPointwise.get_reduction_sizer   c                 C  r   r   rn   rD  rn   rn   ro   r    r   zPointwise.get_reduction_typeoutput_nameindexer!Callable[[Sequence[Expr]], Never]varsry  rs   c                 C  s"   |   }t|p	d||||S Nunnamed)r  rS   storerA  rC  rD  rF  loaderrn   rn   ro   store_output  s   zPointwise.store_outputr   r  rb   c                 C  s.   |   }ttd||}t|| j|| jdS FMove this to a given device. Requires that all reads are to constants.override_devicer   r   r  r	  )r  r   r   ConstantBufferrA  r   r	  rA  r   rK  rn   rn   ro   r    s
   zPointwise.constant_to_deviceNr  r  r  rC  r   rD  rE  rF  ry  rr   rs   r  )rj   rk   rl   r  r  r  rL  r  rn   rn   rn   ro   rA    s    



	rA  c                   @  s6   e Zd ZU ded< dZded< dd
dZdddZdS )Scatterr  output_indexerNr?   scatter_moder   r  rr   rb   c                 C  s6   |   }ttd||}t|| j|| j| j| jdS )rN  rO  )r   r   r  r	  rU  rV  )	r  r   r   rQ  rT  r   r	  rU  rV  rR  rn   rn   ro   r    s   zScatter.constant_to_devicerC  r   rD  rE  rF  ry  rs   c                 C  s6   |   }|d u r
d}tj||| |||| jdS )NrH  )mode)r  rS   rI  rU  rV  rJ  rn   rn   ro   rL    s   zScatter.store_outputr  rS  )rj   rk   rl   rm   rV  r  rL  rn   rn   rn   ro   rT    s
   
 
rT  
logical_ormaximumminimummuladdZbitwise_xor)anyr2  minprodsumxor_sumz"dict[str, Callable[..., OpsValue]]REDUCTION_COMBINE_FNreduction_typearg_break_ties_leftCallable[..., object]c                   sR   t v rt  S dv rd fdd}|S d	kr"ddd}|S td )Nargmaxargminatuple[object, object]brr   tuple[OpsValue, OpsValue]c                   s   | \}}|\}}dkrt ||}nt ||}t ||}trCt ||}t ||}	t |t ||	}t |t ||	} rKt ||nt ||}
t |t ||
}t |||t |||fS )Nrh  )	rS   ltgteqr    nerX  logical_andwhere)ri  rk  Za_valueZa_indexZb_valueZb_indexmaskequalZa_isnanZb_isnanZtierd  r   rc  rn   ro   argmax_combine_fn  s&   
z3get_reduction_combine_fn.<locals>.argmax_combine_fnwelford_combine#tuple[OpsValue, OpsValue, OpsValue]c                 S  sR   | \}}}|\}}}|| }|| }	||	 }
|||
  || || | |
  |	fS r   rn   )ri  rk  Za_meanZa_m2Za_weightZb_meanZb_m2Zb_weightdeltaZ
new_weightZ	w2_over_wrn   rn   ro   welford_combine_fn  s   


z4get_reduction_combine_fn.<locals>.welford_combine_fnzunknown reduction_type=)ri  rj  rk  rj  rr   rl  )ri  rx  rk  rx  rr   rx  )rb  rj  )rc  r   rd  rv  rz  rn   ru  ro   get_reduction_combine_fn  s   
r{  c                      s:  e Zd ZU ded< ded< ded< ded< ddddZeZde fddZdfddZdgddZdhddZ	did!d"Z
djd$d%Zded&d'Zdkd+d,Ze	-dldmd8d9Zednd<d=Zeejd-fdod@dAZedpdDdEZedpdFdGZedqdKdLZedrdQdRZedsdYdZZedtd^d_Zedud`daZedvdbdcZ  ZS )w	Reductionr  reduction_rangesr>   rc  rf  	src_dtyperB   reduction_hintrr   r   c                 C  r  )N)r	  r}  rc  r  rD  rn   rn   ro   r  ,  r  zReduction.__str__r  c                   s"   t   t jdd | jD  B S )Nc                 s  r
  r   r  r  rn   rn   ro   r  3  r  z5Reduction.get_unbacked_symbol_uses.<locals>.<genexpr>)r  r  r+   r  r}  rD  r  rn   ro   r  1  s   z"Reduction.get_unbacked_symbol_usesr  c                 C  rP  r   )r}  rD  rn   rn   ro   r  6  rR  zReduction.get_reduction_sizer   c                 C  rP  r   )rc  rD  rn   rn   ro   r  9  rR  zReduction.get_reduction_typerC  rD  rE  rF  ry  reduction_varsSequence[Symbol]rs   c              	   C  s4   t | j| j| j| ||}t |pd|||S rG  )rS   	reductionr   r~  rc  r  store_reduction)rA  rC  rD  rF  r  r   rn   rn   ro   r  <  s   
zReduction.store_reductionr   c                 C     t | jt | j S r   )r   r	  r}  rD  rn   rn   ro   index_lengthK  r   zReduction.index_lengthSequence[Sequence[Expr]]c                 C  s$   |  | j}|  | jtj}||fS r   )r+  r	  r}  r/   R0_INDEXrA  r   rindexrn   rn   ro   r.  N  s   zReduction.inner_fn_argsc                 C  s*   |  | j}|  | jtj}t| j||S r   )r+  r	  r}  r/   r  r7   r  r  rn   rn   ro   r  S  s   z(Reduction.inner_fn_free_unbacked_symbolsr   r  rb   c              
   C  s>   |   }ttd||}t|| j|| j| j| j| j	t
jdS )rN  rO  r   r   r  r	  r}  rc  r~  r  )r  r   r   rQ  r|  r   r	  r}  rc  r~  rB   DEFAULTrR  rn   rn   ro   r  X  s   zReduction.constant_to_deviceN	dst_dtyper  r   r	  %Union[ReductionType, Literal['scan']]reduction_numelr   
input_noder   tuple[ReductionHint, _IntLike]c	           #   
   C  s  d!dd}	t jj|}
t jjt|}|dkp(t j| tj o(|dvo(tj	}|	|
r1|	|s6t
jd	fS t| }|j}d
}|rWtjt jj| dd}tjt jj| dd}nd"dd}|}|d	kr||
|}|d	krpt
j|fS |d urt|trttdd t|\}}W d    n1 sw   Y  |d ur|d urt jjt|| }|
|krtd||||| t
jdfS t
j|fS |
|ks||d d
 krt
jd	fS t| |||||dkr|nd|t
jd}d#dd}||\}}|r||\}}t|dkrt
jd	fS t|  |! \\}}}d}d}|D ].}t jj"||} t jj#| |t$|% }!t&dd  |!D }"|"rA|d	7 }q|d	7 }q||krTt
j||
|fS t
j'||
|fS )$Nr   r   rr   rh   c                 S  s   t | ttfS r   )ru   r   r   r  rn   rn   ro   
_is_statics  r  z(Reduction.num_splits.<locals>._is_staticscanrf  r0       T)Zinner_reductionFreduction_numel_hintr   
numel_hintc                 S  r  Nr0   rn   )r  r  rn   rn   ro   inner_reduction_splits     z4Reduction.num_splits.<locals>.inner_reduction_splitsr-  zUse previous IRNode's range and reduction_ranges instead of split. current ranges: %s, current reduction ranges: %s, current split: %d, new ranges: %s, new reduction ranges: %sr  r   r`  r  r%  r|  tuple[Sequence[Expr], bool]c                   s   t d t|  |  |  d| d}| }|jd usJ dd |jD }g }d}t|jdd dD ]7 t	 fd	d
|D ri|
 j  jtjjv ritjj j }t|jdd }|  t|jdd |krid}q2||fS )Nr   r   r   r   r)  r(  c                 S  s&   g | ]}t |trt |tjs|qS rn   )ru   r   r}   Numberr   r%  rn   rn   ro   r     s    
zBReduction.num_splits.<locals>.get_read_indices.<locals>.<listcomp>Fc                 S  rP  r   r   r  rn   rn   ro   <lambda>      z@Reduction.num_splits.<locals>.get_read_indices.<locals>.<lambda>keyc                 3  s    | ]	}| j jv V  qd S r   )r   Zfree_symbolsr  mdrn   ro   r        zAReduction.num_splits.<locals>.get_read_indices.<locals>.<genexpr>r   T)ComputedBufferr   r   r   r   r  
range_varssortedr  r  appendr   r   rU   r   Zname_to_bufferr   r)  decide_layout)r%  cbread_writesr  indiceschangedbufZoriginal_stridern   r  ro   get_read_indices  s6   	z.Reduction.num_splits.<locals>.get_read_indicesr   c                 s  s    | ]}|d kV  qdS r0   Nrn   r   rn   rn   ro   r    r  z'Reduction.num_splits.<locals>.<genexpr>)r   r   rr   rh   )r  r   r  r   rr   r   )r%  r|  rr   r  )(rU   r   r   r  rQ   has_featurer3   ZREDUCE_TO_SINGLE_ELEMENTr1   Zsplit_reductionsrB   r  rA   r"  Zmulti_processor_count	functoolsr	   choicesZreduction_split_factorZINNERru   ra   r   r   r   r8   logdebugr|  r   r2   index_vars_squeezer   r  simplify_with_rangesstride_hintsrv   keysr  OUTER)#r   r  r~  r  r	  r}  rc  r  r  r  r  r  Zshould_splitpropsZnum_smZmin_elements_per_threadr  Zouter_reduction_splitssplit
new_rangesnew_reduction_rangesZextracted_numel_hintr%  r  r  r  r   r  Zranges1Z	num_outerZ	num_innerr   jr!  outerrn   rn   ro   
num_splitsg  s   
	











!

zReduction.num_splits<Callable[[Sequence[_IntLike], Sequence[_IntLike]], OpsValue](Callable[[Sequence[_IntLike]], OpsValue]c                   sn   dd D t || d fdd|d	v r3td
d
t dfddfddS S )z1Convert inner_fn from a reduction to an pointwisec                 S     g | ]	}t jj|qS rn   )rU   r   r   Zevaluate_static_shaper   r   rn   rn   ro   r     s    z2Reduction._unroll_reduction_fn.<locals>.<listcomp>r   r  rr   r
   c                   s,   t  fddtjdd D  D S )Nc                 3  s    | ]} |V  qd S r   rn   )r   r  )r   value_fnrn   ro   r    s
    
z=Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<genexpr>c                 S  s   g | ]}t |qS rn   )r   r  rn   rn   ro   r     r   z>Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<listcomp>)r  reduce	itertoolsproductr   )
combine_fnr}  r  r   ro   r     s   z*Reduction._unroll_reduction_fn.<locals>.fnrh  rg  Nr  rl  c                   s*   dd |D }| |t  |tjfS )Nc                 S     g | ]}t |qS rn   )r}   expandr   rn   rn   ro   r   0  r   zDReduction._unroll_reduction_fn.<locals>.value_fn.<locals>.<listcomp>)rS   
index_exprr   int64r   r  )flatten_indexr  rn   ro   r  -  s   z0Reduction._unroll_reduction_fn.<locals>.value_fnc                   s    | d S r  rn   r   )r   rn   ro   r  6  s    z0Reduction._unroll_reduction_fn.<locals>.<lambda>)r   r  rr   r
   )r   r  r  r  rr   rl  )r{  r+  r   r   r  )r  r}  rc  r~  rn   )r  r  r   r  r}  r  ro   _unroll_reduction_fn  s$   
	zReduction._unroll_reduction_fnr  ra   c
                   s  t jjt}
|
dkrDd fdd}|d|d|d|dd	 v s0J  d
d fdd}tj|||t|dS |
dkredv rTd fdd}ndfdd}tj| ||dS t	|
t
rt jj|
tjk rt|dkst|jrtj| | ||dS | | |||
|		\}}|tjkr|}|dkr|	d usJ t|	\}}|d usJ |d usJ | | |||||
S |dkr| | ||||	S tt| |||dS )Nr   valr   rr   Union[bool, float, int]c                   sH    t jkr	t| S  jrt| tjsJ t| S t| tjs J t| S r   )	r   rh   r=  ru   typingSupportsFloatr?  SupportsIntr   r  r  rn   ro   py_cnstN  s   
z!Reduction.create.<locals>.py_cnstr0   )r`  ra  r_  r]  z* not supported for zero-dimension tensors!r   r   rT   c                   s   t   S r   rS   r>  r   )r  rc  rtypes_to_initsrn   ro   const_fnd     z"Reduction.create.<locals>.const_fnrP  r  c                   s   t d S r  r  r   r  rn   ro   r   r  r   zReduction.create.<locals>.fnc                      dd D } | |S )Nc                 S     g | ]}t jjqS rn   r}   r'  r(  r   rn   rn   ro   r   x  r   z0Reduction.create.<locals>.fn.<locals>.<listcomp>rn   r   reduction_index)r  r}  rn   ro   r   w     
r  r  )r  r   rr   r  )r   r   rr   rT   )rU   r   r   simplifyrQ   r  rA  r"  rv   ru   r   r   r1   Zunroll_reductions_thresholdrM   r   r  r  rB   r  r8   !create_multilayer_existing_rangescreate_multilayerra   r|  )r#  r   r  r~  r  r	  r}  rc  r  r  r  r  r  r   hintr  r  r  rn   )r  r  r}  rc  r  ro   r"  ;  s   

zReduction.creater   #Union[_NumLike, Sequence[_NumLike]]c                 C  s   | dv rt |rtdS t|rdS t|jS | dv r0t |r$tdS t|r*dS t|jS t|r6dnd}t|r>dnd}|||||||f|||ftd|fd	|  S )
N)r2  rg  z-infF)r^  rh  infTr   r0   )r`  r_  ra  r]  welford_reducerw  online_softmax_reduce)r    r?  r   r   iinfor^  r2  )rc  r   zeroonern   rn   ro   default_accumulator  s0   
zReduction.default_accumulatorc                 C  s   | dkrdS t | |S )Nr  r   )r|  r  rc  r   rn   rn   ro   default_value  s   zReduction.default_valuer  r]   r  c                 C  sP   | dkr|S | dkr|dkr|t jkrt jS | dkr&|dkr&|t jkr&t jS |S )Nr     i      )rB   r  Z
OUTER_TINY)r  r  r  rn   rn   ro   _multilayer_second_step_hint  s   
z&Reduction._multilayer_second_step_hintrK  
block_sizedefaultre  c                   sF   t |gtjjt| d d	 fdd}|S )
Nr   r   r  r  rr   rT   c                   sh   |\}| ^ }| |  d fdd}r1t t  tjt tj}t ||S | S )Nrr   rT   c                     s    gS r   rn   rn   )r  rK  	new_indexr   rn   ro   body  r  zCReduction._multilayer_wrap_loader.<locals>.wrapper_fn.<locals>.body)rr   rT   )rS   rm  r  r   Zint32Zmasked)r   r  Zreduction_blockr  rs  r  r  rK  	need_maskr  r   )r  r  ro   
wrapper_fn  s   
z5Reduction._multilayer_wrap_loader.<locals>.wrapper_fn)r   r  r  r  rr   rT   )Viewdynamic_reshape_indexerrU   r   r   r  r}   r  )r#  rK  r}  r  r  r  r  r  rn   r  ro   _multilayer_wrap_loader  s   
z!Reduction._multilayer_wrap_loader@Callable[[Sequence[sympy.Expr], Sequence[sympy.Expr]], OpsValue]original_rangesoriginal_reduction_rangesr  Sequence[Integer]r  c                   sN   t dd D sJ dt|t|t| d fd	d
}|S )Nc                 s  s    | ]}|d kV  qdS r  rn   r  rn   rn   ro   r  4  r  zDReduction._multilayer_wrap_loader_existing_ranges.<locals>.<genexpr>z8Only enabled for numel_hint == 1, found original_ranges=merged_indexr  new_reduction_indexrr   rT   c                   s:   | d t  }| t d  } |t|t| S r   )r   rw   )r  r  Zoriginal_idxr  rK  r  r   rn   ro   r  ;  s   zEReduction._multilayer_wrap_loader_existing_ranges.<locals>.wrapper_fn)r  r  r  r  rr   rT   )r  r  r  rw   )r#  rK  r  r  r  r  r  rn   r  ro   '_multilayer_wrap_loader_existing_ranges+  s   	z1Reduction._multilayer_wrap_loader_existing_rangesr  
list[Expr]list[Integer]c                   s   |t jt jfvr
|nt j}t|||||||	|}|  |  d
 fdd}tj	j
t|}| |
||}||dt| ksDJ tt|||||t|d |	||d	S )a
        Break a large reduction up into multiple smaller reductions
        recursively
        r   r  r  rr   rT   c                   s    g | |S r   rn   r  Zintermediate_loaderrn   ro   intermediate_fnp  s   z;Reduction.create_multilayer_helper.<locals>.intermediate_fnNr  )r   r  r  r  rr   rT   )r   Zfloat16Zbfloat16r?  r|  r"  r  r  rU   r   r   r   rQ   r  r   ra   )r#  r   r  r~  r  r  r  r  r  rc  r  r  Zintermediate_dtypeZintermediater  r  rn   r  ro   create_multilayer_helperH  sD   
z"Reduction.create_multilayer_helperc
                 C  sb   t |}
t|
|d  |}| ||}| |||
|||}| ||||||g |||g|||	S )r
  r0   )rQ   r-   r  r  r  )r#  r   r  r~  r  r	  r}  rc  r  r  r  r  r  r  rn   rn   ro   r    s&   
zReduction.create_multilayerc                 C  s8   |  |||||}| ||||||g ||||	d|
S )r
  r  )r  r  )r#  r   r  r~  r  r  r  r  r  rc  r  r  rn   rn   ro   r    s(   
z+Reduction.create_multilayer_existing_rangesr  r  r  r  
rC  r   rD  rE  rF  ry  r  r  rr   rs   r  rr   r  r  r   )r   r  r  rf  r~  rf  r  r   r	  r  r}  r  rc  r  r  r   r  r   rr   r  )
r  r  r}  r  rc  r   r~  rf  rr   r  )r   r  r  rf  r~  rf  r  r  r	  ry  r}  ry  rc  r>   r  rB   r  r   rr   ra   rc  r   r   rf  rr   r  )r  r]   r  r   r  rB   rr   rB   )rK  r   r}  r  r  r]   r  r]   r  r]   r  r  rr   re  )rK  r   r  ry  r  ry  r  r  r  r  rr   r   )r   r  r  rf  r~  rf  r  r  r  ry  r  ry  r  r  r  r	  rc  r>   r  r]   r  rB   rr   ra   )r   r  r  rf  r~  rf  r  r  r	  ry  r}  ry  rc  r>   r  r]   r  rB   rr   ra   )r   r  r  rf  r~  rf  r  r  r  ry  r  ry  r  r	  r  r	  rc  r>   r  rB   rr   ra   )rj   rk   rl   rm   r  r7  r  r  r  r  r  r.  r  r  r  r  r  r8  rB   r  r"  r  r  r  r  r  r  r  r  r:  rn   rn   r  ro   r|  $  sT   
 








 $/
 #?&r|  c                      s2   e Zd ZU ded< d fddZd ddZ  ZS )!MultiOutputReductionr   output_indexr   r  r  rf  	inner_fns)Union[INNER_FN_TY, Sequence[INNER_FN_TY]]r	  r  r}  rc  r>   r~  r  rB   c
              
     sX   t  r f t dkr d }
nd fdd	}
t j|||
|||||d
 |	| _d S )Nr0   r   r   ry  reduction_idxrr   tuple[OpsValue, ...]c                   s   t  fddD S )Nc                 3  s    | ]}| V  qd S r   rn   r   r   r   r  rn   ro   r        z@MultiOutputReduction.__init__.<locals>.loader.<locals>.<genexpr>)rw   r  r  r  ro   rK    s   z-MultiOutputReduction.__init__.<locals>.loaderr  )r   ry  r  ry  rr   r  )callabler   r  __init__r  )rA  r   r  r  r	  r}  rc  r~  r  r  rK  r  r  ro   r    s    


zMultiOutputReduction.__init__rC  r   rD  rE  rF  ry  r  r  rr   rs   c              	   C  sZ   t | j| j| j| ||}t|ttfsJ t	| || j
 }t |p'd|||S rG  )rS   r  r   r~  rc  r  ru   rw   rv   r   r  r  )rA  rC  rD  rF  r  ry   r   rn   rn   ro   r    s   

z$MultiOutputReduction.store_reduction)r   r  r  rf  r  r  r	  r  r}  r  rc  r>   r~  rf  r  rB   r  r   r  )rj   rk   rl   rm   r  r  r:  rn   rn   r  ro   r    s   
 %r  c                   @  s"   e Zd ZeejdfdddZdS )OnlineSoftmaxReductionNr   r  r  rf  r~  r  r  r	  ry  r}  
num_outputr   r  rB   r  r   rr   Sequence[TensorBox]c
                   s<   t  fddt|D }
|
D ]}|  q|
S )z>
        Create the reduction disregarding splitting.
        c                 3  s.    | ]}t t d |	V  qdS )r  N)ra   r"  r  r   Z
output_idxr   r  r  r	  r  r}  r~  rn   ro   r  %  s"    
z0OnlineSoftmaxReduction.create.<locals>.<genexpr>)rw   r   r  )r#  r   r  r~  r  r	  r}  r  r  r  resultsr   rn   r!  ro   r"    s   
zOnlineSoftmaxReduction.create)r   r  r  rf  r~  rf  r  r  r	  ry  r}  ry  r  r   r  rB   r  r   rr   r  )rj   rk   rl   r8  rB   r  r"  rn   rn   rn   ro   r    s
    
r  c                   @  s<   e Zd ZeejfdddZedddZedddZ	dS )WelfordReductionr   r  r   rf  r  Sequence[Callable[..., Any]]r	  r	  r}  rc  r>   r  rB   rr   r  c              
     s6  dv sJ t jjt}dfdd}	|dkr-|	d}
|	d}|	d}|
||fS |d	krWdfdd dkrL d |	d|	d	fS t fddD S tjd |d\}}tj	kro||d	kr| 
|S fddtdD }|D ]}|  q|S )N)r  rw  r  r   rr   ra   c                   s&   d fdd}t j|tdS )	Nr   ry  rr   rT   c                   s   t  S r   r  r   )r   r  rn   ro   r  K  s   z8WelfordReduction.create.<locals>.const.<locals>.inner_fnrP  r   ry  rr   rT   rA  r"  rv   )r  r  )r   r   r	  r  ro   constJ  s   z&WelfordReduction.create.<locals>.constr   r0   rK  4Callable[[Sequence[Expr], Sequence[Expr]], OpsValue]c                   s&   d fdd}t j|tdS )	Nr   ry  rr   rT   c                   r  )Nc                 S  r  rn   r  r   rn   rn   ro   r   d  r   zKWelfordReduction.create.<locals>.copy.<locals>.inner_fn.<locals>.<listcomp>rn   )r   r  )rK  r}  rn   ro   r  c  r  z7WelfordReduction.create.<locals>.copy.<locals>.inner_fnrP  r&  r'  )rK  r  )r   r   r	  r}  rK  ro   copy`  s   z%WelfordReduction.create.<locals>.copyr  c                 3  s    | ]} |V  qd S r   rn   r  )r+  rn   ro   r  q  r  z*WelfordReduction.create.<locals>.<genexpr>)rc  r  c                   s*   g | ]}t t |	qS rn   )ra   r"  r#  r   )r   r   r  r	  r  r}  rc  rn   ro   r     s     z+WelfordReduction.create.<locals>.<listcomp>r   )r  r   rr   ra   )rK  r)  rr   ra   )rU   r   r   r  rQ   rw   r|  r  rB   r  r  r   r  )r#  r   r   r  r	  r}  rc  r  r  r(  meanm2weightr  r  r"  r   rn   )r+  r   r   r  r	  r  r}  rc  ro   r"  ;  sT   



zWelfordReduction.creater   r  c                 C  r  )N)r   r   r   rn   r  rn   rn   ro   r    r  zWelfordReduction.default_valuer  r]   c	              
     s$  t tjjt d }	|	r9|dkr9dfd
d}
j||d t|
ddt|
ddf|d|dS t	d   t
|t fdd|D g | g||}|D ]}|  qadddtjjt |}||}t
|tfdd|D |gd|S )r
  r   rw  r   ry  r  r   r   rr   rT   c                   s   t | S r   r  )r   r  r   rg  rn   ro   r>    s   z4WelfordReduction.create_multilayer.<locals>.constantr   r0   )r   r   r  r	  r}  rc  r  r  c              	   3  s&    | ]}j | d dV  qdS )r   )r  N)r  )r   rK  )r  r#  r  r}  r  rn   ro   r    s    	
z5WelfordReduction.create_multilayer.<locals>.<genexpr>r   r  rK  r  c                 S  s   |g | |S r   rn   )r   r  rK  rn   rn   ro   intermediate_loader_fn  s   zBWelfordReduction.create_multilayer.<locals>.intermediate_loader_fnc                 3  s     | ]}t  | d V  qdS )r*  N)r	   r  r   )r/  rn   ro   r    
    
N)r   ry  r  ry  r   r   rr   rT   )r   ry  r  ry  rK  r  rr   rT   )rQ   rU   r   r   r  r}   r  r  r	   r-   r#  r"  rw   r  r   r  )r#  r   r   r  r	  r}  rc  r  r  r  r>  Zintermediatesr   r  rn   )r  r#  r   r/  r  r}  r  ro   r    sb   

	


z"WelfordReduction.create_multilayerN)r   r  r   rf  r  r$  r	  r	  r}  r	  rc  r>   r  rB   rr   r  r  )r   r  r   rf  r  r$  r	  r	  r}  r	  rc  r>   r  r]   r  rB   rr   r  )
rj   rk   rl   r8  rB   r  r"  r  r  r  rn   rn   rn   ro   r#  :  s    	xr#  c                      s   e Zd ZU ded< ded< ded< ded< ded	< d
ed< ded< ded< dH fddZdI fddZdJdd ZdKd!d"ZdLd$d%ZdMd&d'Z	dMd(d)Z
dNd*d+ZdOd-d.ZdHd/d0Zeejfd1d2dPd<d=ZedQdFdGZ  ZS )RScanr	  scan_rangesr   =Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]r  zFCallable[[Sequence[_IntLike], Sequence[_IntLike]], Sequence[_IntLike]]r   rB   r  r   r  tuple[torch.dtype, ...]dtypestuple[Callable[..., Any], ...]r  rr   r  c                   :   t   t jdd | jD  B t jdd | jD  B S )Nc                 s  r
  r   r  r  rn   rn   ro   r  ,  r  z0Scan.get_unbacked_symbol_uses.<locals>.<genexpr>c                 s  r
  r   r  r  rn   rn   ro   r  -  r  )r  r  r+   r  r2  r   rD  r  rn   ro   r  &  s   zScan.get_unbacked_symbol_usesrs   c                   0   t | jt | j t | jksJ t   d S r   )r   r	  r2  r   r  rE  rD  r  rn   ro   rE  0     "zScan.__post_init__rC  r   rD  %Callable[[Sequence[_IntLike]], Never]rF  ry  	scan_varsr  c                   sR   |  || t fdd| jD }t| j| j|}t|p d| || j S )Nc                 3      | ]}| V  qd S r   rn   r   r  r%  rn   ro   r  <  r  z'Scan.store_reduction.<locals>.<genexpr>rH  )	r   rw   r  rS   r  r5  r  rI  r  )rA  rC  rD  rF  r;  ry   resultrn   r%  ro   r  4  s   zScan.store_reductionc                 C  r  )NZcustomrn   rD  rn   rn   ro   r  B     zScan.get_reduction_typer  c                 C  rP  r   )r2  rD  rn   rn   ro   r  F  rR  zScan.get_reduction_sizec                 C  rP  r   r   rD  rn   rn   ro   r   I  rR  zScan.get_sizec                 C  rP  r   r  rD  rn   rn   ro   r   L  rR  zScan.get_pointwise_sizec                 C  r  r   )r   r	  r2  rD  rn   rn   ro   r  O  r   zScan.index_lengthr1  c                 C  .   |  | j}|  | jtj}| ||}|fS r   )r+  r	  r2  r/   r  r   rA  r   r  r   rn   rn   ro   r.  R     zScan.inner_fn_argsc                 C  4   |  | j}|  | jtj}| ||}t| j|S r   )r+  r	  r2  r/   r  r   r7   r  rB  rn   rn   ro   r  X     z#Scan.inner_fn_free_unbacked_symbolsT)can_fallback_to_atenr   r  +tuple[Callable[[Sequence[Expr]], Any], ...]axisrF  rh   r   r
   Sequence[Optional[TensorBox]]c                  s  g d    d d    g	t jtjs$d gt S tdkr9t jtjs9d gt S t jj}
|
t		}ttksNJ |

t|drgfddttD S | jd d  	|d\}t
|dkrtjjd u ptotdkotdk}|s|rd gt S d}nt
d 	fdd	
fddttD }|D ]}|  q|S )Nr0   c                   &   g | ]}t j | | d qS rP  rA  r"  r   r  r   r5  r  r   rn   ro   r         zScan.create.<locals>.<listcomp>r   )r   r   r  rH  pointwise_rangesr2  r  
scan_numelz3.3.0r   ry  
scan_indexrr   r  c                   H   t |t ks
J t | t ksJ g | d   ||  d  S r   r   )r   rR  )rH  rP  r2  rn   ro   r         zScan.create.<locals>.reindexc                   sB   g | ]}t 	d| | 
 |d qS ))r   r   r5  r  r  r   r	  r2  r  r   r  r  rn   )ra   r"  rM  )r  r   r5  r  r   rP  r  r   r2  	scan_typer   rn   ro   r     s*    )r   ry  rR  ry  rr   r  )rU   r   r  r3   ZSCANr   ZTUPLE_REDUCTIONr   r  rQ   r  r}   Ler   r  r1  r   versionZhip
has_tritontriton_version	SplitScanr  )r#  r   r5  r  r   rH  r  r  rF  r   r   rQ  r  Zsupports_splitr"  r>  rn   )rH  r  r   r5  r  r   rP  r  r   r2  rV  r   ro   r"  ^  sV    







zScan.creater   rf  r  r  rP  rQ  r   r  c	           
   
     s*   d
 fdd}	t j||||	||d|d	S )Nr   ry  r  rr   rT   c                   s$   g | d   ||  d  S r   rn   r  rH  r  rn   ro   r    s   $z#Scan.num_splits.<locals>.wrapper_fnr  )r   r  r~  r  r	  r}  rc  r  )r   ry  r  ry  rr   rT   )r|  r  )
r#  r   r   r  rH  rP  r2  r  rQ  r  rn   r\  ro   r    s   zScan.num_splitsr  r  )
rC  r   rD  r:  rF  ry  r;  r  rr   rs   r  r  r  r  r6  )r   r  r5  r4  r  rG  r   r	  rH  r   r  r3  r  rB   rF  rh   r   r
   rr   rI  )r   r  r   rf  r  r  rH  r   rP  r	  r2  r	  r  r3  rQ  r   rr   r  )rj   rk   rl   rm   r  rE  r  r  r  r   r   r  r.  r  r8  rB   r  r"  r  r:  rn   rn   r  ro   r1    s4   
 








	ar1  c                   @     e Zd ZdS )r[  Nrj   rk   rl   rn   rn   rn   ro   r[    s    r[  c                      s   e Zd ZU ded< ded< ded< ded< ded	< d
ed< ded< ded< ded< d9 fddZd: fddZd;dd Zd<d!d"Zd=d#d$Zd=d%d&Z	d=d'd(Z
d>d)d*Zd?d,d-Zd9d.d/Zeejfd@d7d8Z  ZS )ASortr	  sort_rangesr   z:Callable[[Sequence[Expr], Sequence[Expr]], Sequence[Expr]]r   rB   r  r   r  r4  r5  r6  r  rh   stable
descendingrr   r  c                   r7  )Nc                 s  r
  r   r  r  rn   rn   ro   r    r  z0Sort.get_unbacked_symbol_uses.<locals>.<genexpr>c                 s  r
  r   r  r  rn   rn   ro   r    r  )r  r  r+   r  r`  r   rD  r  rn   ro   r    s   zSort.get_unbacked_symbol_usesrs   c                   r8  r   )r   r	  r`  r   r  rE  rD  r  rn   ro   rE    r9  zSort.__post_init__rC  r   rD  r  rF  ry  r  c                   sV   |  || t fdd| jD }t| j|| j| j}t|p"d| || j	 S )Nc                 3  r<  r   rn   r=  r%  rn   ro   r  	  r  z'Sort.store_reduction.<locals>.<genexpr>rH  )
r   rw   r  rS   sortr5  ra  rb  rI  r  )rA  rC  rD  rF  r  ry   r>  rn   r%  ro   r    s   zSort.store_reductionc                 C  r  )Nrc  rn   rD  rn   rn   ro   r  	  r   zSort.get_reduction_typec                 C  rP  r   )r`  rD  rn   rn   ro   r  	  rR  zSort.get_reduction_sizec                 C  rP  r   r@  rD  rn   rn   ro   r   	  rR  zSort.get_sizec                 C  rP  r   r  rD  rn   rn   ro   r   	  rR  zSort.get_pointwise_sizec                 C  r  r   )r   r	  r`  rD  rn   rn   ro   r  	  r   zSort.index_lengthr  c                 C  rA  r   )r+  r	  r`  r/   r  r   rB  rn   rn   ro   r.  	  rC  zSort.inner_fn_argsc                 C  rD  r   )r+  r	  r`  r/   r  r   r7   r  rB  rn   rn   ro   r  !	  rE  z#Sort.inner_fn_free_unbacked_symbolsr   r  'tuple[Callable[[list[Expr]], Any], ...]rH  r   r
   rI  c	                   s*  g 	d   	 d d  	  g
t jtjs$d gt S t jj}
|
t
}d}t	j
jo=|
t||}|sGd gt S ttksQJ |
t|drj	fddttD S d 
fd
d	
fddttD }|D ]}|  q|S )Nr0   r  c                   rJ  rK  rL  rM  rN  rn   ro   r   L	  rO  zSort.create.<locals>.<listcomp>r   ry  
sort_indexrr   r  c                   rS  r   rT  )r   re  )rH  rP  r`  rn   ro   r   V	  rU  zSort.create.<locals>.reindexc                   sD   g | ]}t td| | 	|
 d qS ))r   r   r5  r  r  r   r	  r`  r   r  r  ra  rb  rn   )ra   r"  r_  rM  )rb  r   r5  r  r   rP  r  r   r   r`  ra  rn   ro   r   [	  s,    )r   ry  re  ry  rr   r  )rU   r   r  r3   ZSORTr   r   r  rQ   r1   r  Zpersistent_reductionsr  r}   rW  r   r  )r#  r   r5  r  r   rH  ra  rb  r  r   r   Z
sort_numelZ
max_rblockZis_persistent_kernelr"  r>  rn   )rH  rb  r   r5  r  r   rP  r  r   r   r`  ra  ro   r"  '	  s0    




zSort.creater  r  )
rC  r   rD  r  rF  ry  r  ry  rr   rs   r  r  r  r  )r   r  r5  r4  r  rd  r   r	  rH  r   ra  rh   rb  rh   r  rB   r   r
   rr   rI  )rj   rk   rl   rm   r  rE  r  r  r  r   r   r  r.  r  r8  rB   r  r"  r:  rn   rn   r  ro   r_    s.   
 








r_  c                 C  s(   z	t | dd W dS  ty   Y dS w )NFfreezeT)r*  rj  r  rn   rn   ro   r   w	  s   r   c                 C  s@   zt | dd\}}| r|  | W S  ty   Y dS w NFrf  )r*  should_pad_stridespad_stridesis_contiguousrj  )r   _bufferr)  rn   rn   ro    is_contiguous_storage_and_layout	  s   
rm  rg  want_contiguousstride_order'Optional[Sequence[Union[int, Integer]]]r  r  tuple[StorageBox, Layout]c           	      C  s   t | trt| j|||||dS t | tr)t| j|||||d\}}| | j fS t | trc|r[|r?|   |   s>J n|durK| j	||d n|durW| j
||d n|   t| |  fS t | trvt| j|d\}}|| jfS t)z
    Try to simplify x into a StorageBox and a Layout.

    allow_padding only affect how we apply stride_order. When allow_padding
    is True, we have the freedom to add padding when applying the stride_order.
    rg  rn  ro  r  r  Nr  rf  )ru   ra   r*  r(  
StorageBoxr   Bufferr  rk  r  r  r  r-  r)  rj  )	r   rg  rn  ro  r  r  r   r)  bufferrn   rn   ro   r*  	  sR   






r*  c                 C  s2   zt | dd\}}||W S  ty   Y dS w rh  )r*  is_stride_orderedrj  )r   ro  rl  r)  rn   rn   ro   "is_stride_order_storage_and_layout	  s   rx  c                   @  s   e Zd ZU ded< dBddZdCdd	ZdDddZdEddZedFddZ	dGddZ
dHddZdIddZdJddZdKd d!ZdLd%d&ZdMd(d)ZdNd+d,Zd-d. Zd/d0 ZdMd1d2ZdMd3d4ZdOd6d7ZdPd9d:Zd;d< ZdQd?d@ZdAS )RBaseViewrb   r(  rr   r  c                 C  
   | j  S r   r(  r  rD  rn   rn   ro   r  	  r  z!BaseView.get_unbacked_symbol_uses*Callable[[Sequence[Expr]], Sequence[Expr]]c                 C  s   t d|  )Nzmake_reindexer NYI on r  rD  rn   rn   ro   make_reindexer	  r  zBaseView.make_reindexerr  c                   &   | j   |  d fdd}|S )Nr   ry  rr   r   c                   r   r   rn   r%  innerr   rn   ro   rD  	  r   z&BaseView.make_indexer.<locals>.indexer)r   ry  rr   r   )r(  r  r}  rA  rD  rn   r  ro   r  	     
zBaseView.make_indexerr  c                   r~  )Nr   ry  rr   rT   c                   r   r   rn   r%  r  rn   ro   rK  	  r   z$BaseView.make_loader.<locals>.loaderr&  )r(  r  r}  rA  rK  rn   r  ro   r  	  r  zBaseView.make_loaderrf  c                 C  rz  r   )r(  r   rD  rn   rn   ro   r   	     
zBaseView.dtyperm  c                 C  rz  r   r(  r   rD  rn   rn   ro   r   	  r  zBaseView.get_layoutr  c                 C  rz  r   r(  r   rD  rn   rn   ro   r   	  r  zBaseView.get_devicer9  c                 C  r   r   rn   rD  rn   rn   ro   rT  	  r   zBaseView.get_origin_noder   c                 C  rz  r   r(  r  rD  rn   rn   ro   r  	  r  zBaseView.get_namery  c                 C  rs  r   r}  rD  rn   rn   ro   r   	  ru  zBaseView.get_pointwise_sizer  r   rs   c                 C     | j |S r   r(  r  r  rn   rn   ro   r   
  r   zBaseView.mark_reuserh   c                 C  rz  r   r(  r  rD  rn   rn   ro   r  
  r  zBaseView.has_exceeded_max_readsr   c                 C  rz  r   r(  r  rD  rn   rn   ro   r  
  r  zBaseView.realizec                 C  rz  r   r(  r  rD  rn   rn   ro   r  	
  r  zBaseView.realize_hintc                 C  rz  r   r(  r  rD  rn   rn   ro   r  
  r  zBaseView.get_storage_numelc                 C  rz  r   r(  r  rD  rn   rn   ro   r  
  r  zBaseView.is_externc                 C  rz  r   )r(  is_module_bufferrD  rn   rn   ro   r  
  r  zBaseView.is_module_bufferrF  c                 C  rz  r   r(  rN  rD  rn   rn   ro   rN  
  r  zBaseView.get_read_namesr  c                 C  sF   t tdd t|  |  jW  d    S 1 sw   Y  d S r,  )r   r   r   r9   r  r   r  rD  rn   rn   ro   rM  
  s   $zBaseView.get_readsc                 C  s"   | }t |tr|j}t |ts|S r   )ru   ry  r(  )rA  r   rn   rn   ro   r  
  s
   

zBaseView.unwrap_viewr   r  c                 C  s2   |   }ttd||}t||  ||  dS rM  )r  r   r   rQ  rA  r   r   rR  rn   rn   ro   r  %
  s   zBaseView.constant_to_deviceNr  )rr   r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )rj   rk   rl   rm   r  r}  r  r  r  r   r   r   rT  r  r   r  r  r  r  r  r  r  rN  rM  r  r  rn   rn   rn   ro   ry  	  s0   
 



		











ry  c                   @  s@   e Zd ZU ded< edd Zedd Zdd	d
Zdd Z	dS )rz   r  r   c                 C  s   t jj}tttj|}|  }dgt|t|  t| }t|t|ks)J t	t|D ]A}|| dkrF|| dus?J || ||< q/|| du s\t jjj
jt|| dddr]q/|j|| ||  dddkspJ dq/|S )	zReplace `-1` with correct sizesNr  r0   TZsize_obliviousr   fallbackzKBroadcast failed in ExpandView({x.get_size()}, {new_size}) on dimension {i})rU   r   r   rv   ra  r}   r  r   r   r   r   evaluate_exprr  r   )r   new_sizer   old_sizer   rn   rn   ro   _normalize_size5
  s"    zExpandView._normalize_sizec           
      C  s   |  ||}t|r\t|\}}t|t|j }|dksJ tjjg| }t|j	|jD ]\}}|
tjjjjt|dddsC|ntjj q-t|j|jt|||j}	t||	dS t||dS )Nr   r0   Tr  r'  )r(  r   )r  r   r*  r   r   r}   r'  r(  r   r   r  rU   r   r   r   r  r  r+  r   r   rv   r,  r-  rz   )
r#  r   r  r.  r/  skipr0  r   r   r1  rn   rn   ro   r"  P
  s.   
zExpandView.createrr   ry  c                 C  rP  r   r@  rD  rn   rn   ro   r   l
  rR  zExpandView.get_sizec                   s4   |   }| j   t|t   fdd}|S )Nc                   sR   t | d  } t| t ksJ tt D ]} | dkr&tjj| |< q| S r  )rv   r   r   r}   r'  r(  )r   r   actualr  rn   ro   r   t
  s   z*ExpandView.make_reindexer.<locals>.reindex)r   r(  r   )rA  targetr   rn   r  ro   r}  o
  s
   
	zExpandView.make_reindexerNr  )
rj   rk   rl   rm   r  r  r8  r"  r   r}  rn   rn   rn   ro   rz   1
  s   
 


rz   c                   @  s@   e Zd ZU ded< edd Zedd Zdd	d
Zdd ZdS )PermuteViewr  dimsc                   s   |  |}t|ttt|ksJ t|r<t|\} t j j fdd|D  fdd|D  j	}t
||dS t||dS )Nc                      g | ]} j | qS rn   r@  r   r/  rn   ro   r   
  r   z&PermuteView.create.<locals>.<listcomp>c                   r  rn   r   r   r  rn   ro   r   
  r   r'  )r(  r  )_map_neg_dimsr+   r   r   r   r*  r+  r   r   r,  r-  r  )r#  r   r  r.  r1  rn   r  ro   r"  
  s   
zPermuteView.createc                   s    fdd D S )Nc                   s$   g | ]}|d kr
|nt  | qS r   rT  )r   r  r  rn   ro   r   
     $ z-PermuteView._map_neg_dims.<locals>.<listcomp>rn   )r#  r  rn   r  ro   r  
     zPermuteView._map_neg_dimsrr   ry  c                   sD   t | | jt tt| jksJ | j   fdd| jD S )Nc                   r   rn   rn   r   r@  rn   ro   r   
  r   z(PermuteView.get_size.<locals>.<listcomp>)r+   r  r  r   r   r(  r   rD  rn   r@  ro   r   
  s
   

zPermuteView.get_sizec                   s^   dd t | jD   fddtt| jD  t ttt| jks'J  fdd}|S )Nc                 S  r   rn   rn   )r   r   r  rn   rn   ro   r   
  r   z.PermuteView.make_reindexer.<locals>.<dictcomp>c                   r   rn   rn   r   invrn   ro   r   
  r   z.PermuteView.make_reindexer.<locals>.<listcomp>c                   s    fddD S )Nc                   r   rn   rn   r   r   rn   ro   r   
  r   z?PermuteView.make_reindexer.<locals>.reindex.<locals>.<listcomp>rn   r   r  r   ro   r   
  r  z+PermuteView.make_reindexer.<locals>.reindex)r   r  r   r   r+   )rA  r   rn   r  ro   r}  
  s
   zPermuteView.make_reindexerNr  )	rj   rk   rl   rm   r8  r"  r  r   r}  rn   rn   rn   ro   r  
  s   
 


r  c                   @  s6   e Zd ZeddddZedddZdddZdS )SqueezeViewNr  c                  s>  t |rst|\}}g }g } d ur(t tsJ dd kr& t|jk s(J tt|j|jD ]0\}\}}	 d u rJ|dkrI|	| |	|	 q1| krY|	| |	|	 q1|dksaJ dq1t
|j|j|||j}
t||
dS  d u rt|dd | D S |   dksJ t| fddt| D S )	Nzexpected integer dim argumentr   r0   zexpected squeezed size to be 1r'  c                 S     g | ]}|d kr|qS r&  rn   r   rn   rn   ro   r   
  r   z&SqueezeView.create.<locals>.<listcomp>c                   s   g | ]
\}}| kr|qS rn   rn   r   r   r   r  rn   ro   r   
      )r   r*  ru   r   r   r   r   r   r   r  r+  r   r   r,  r-  r  r"  r   )r#  r   r  r.  r/  r  r0  r   r   r   r1  rn   r  ro   r"  
  s:   


"zSqueezeView.creater   r  c                   s@   dd | D }dd t | D t|  d
 fdd	}||fS )Nc                 S  r  r&  rn   r   rn   rn   ro   r   
  r   z(SqueezeView.squeezer.<locals>.<listcomp>c                 S  s   g | ]
\}}|d kr|qS r&  rn   r  rn   rn   ro   r   
  r  r   list[sympy.Expr]rr   tuple[sympy.Expr, ...]c                   sT   t | t ksJ |  d tjjg  }t| D ]\}}|||< qt|S )N )r   r}   r'  r(  r   rw   )r   r  r   r   lengthZnot_onern   ro   r   
  s
   "
z%SqueezeView.squeezer.<locals>.reindex)r   r  rr   r  )r   r   )r   r  r   rn   r  ro   squeezer
  s
   zSqueezeView.squeezerrr   rs   c                 C  s   t d)Nzuse SqueezeView.create())AssertionError)rA  r(  rn   rn   ro   r  
  ru  zSqueezeView.__init__)r   r  r  )rj   rk   rl   r8  r"  r  r  r  rn   rn   rn   ro   r  
  s    %r  c                   @  sT   e Zd ZU ded< ded< dd Zdd	d
ZdddZeZedd Z	dddZ
dS )GenericViewr  r   r  r   c                 C  rP  r   )r   rD  rn   rn   ro   r}  
  rR  zGenericView.make_reindexerrr   r   c                 C  sB   dd t t| jD }t| |}ddtt| d| S )Nc                 S     g | ]}t tj|qS rn   )rP   r/   r9  )r   r)  rn   rn   ro   r   
      z+GenericView.reindex_str.<locals>.<listcomp>zlambda , r   )r   r   r   rv   r   rc  ra  r   )rA  Z	index_oldZ	index_newrn   rn   ro   reindex_str
  s
   zGenericView.reindex_strc                 C  s$   |  | jd| j d|   gS )Nsize=zreindex=)re  r(  r   r  rD  rn   rn   ro   r  
  s   zGenericView.__str__c                 C  s   | |t ||dS )Nr(  r   r   )rv   )r#  r   r  r   rn   rn   ro   r"  
  r  zGenericView.createry  c                 C  rP  r   r@  rD  rn   rn   ro   r     rR  zGenericView.get_sizeNr  r  )rj   rk   rl   rm   r}  r  r  r7  r8  r"  r   rn   rn   rn   ro   r  
  s   
 


r  c                   @  sH   e Zd Zedd Zedd Zedd Zedd Zed	d
 Z	dS )r  c                 C  s<   t | } t |}tjjjj}|t | dr| | } | S r  )r}   r  rU   r   r   r   r  Lt)r   r   r  rn   rn   ro   handle_negative_index  s   

zView.handle_negative_indexc           	        s  t |ttfs	J | | |\ }tjj |r|S d}t	t
 dks/t	t
|dkr1d}d|v rD fdd}| |t||dS t|sJ|rw|r[t|s[t|t| }t|dd\}}t|j|j|t||j}t||dS |  |}| |t||dS )	NFr   Tc                   s   t dgt  S r  )rw   r   r   r  rn   ro   fake_reindex"  r  z!View.create.<locals>.fake_reindexr  )rn  r'  )ru   rw   rv   resolve_negative_sizer   rU   r   r   Zstatically_known_list_equalsr   r%   rm  ExternKernelrequire_exact_stridesr   r   r*  r+  r   r   r,  r-  r  )	r#  r   r  Zunbacked_symbols_in_sizesr  r.  r/  r1  r   rn   r  ro   r"    s6   zView.createc                 C  s   dd |D }dd | D } t |}tt|D ]}|| dkr3tjj||< tt| t|||<  nqtj	j
t| t| | |fS )Nc                 S  r  rn   rU   r   r   r  r  rn   rn   ro   r   A      z.View.resolve_negative_size.<locals>.<listcomp>c                 S  r  rn   r  r  rn   rn   ro   r   B  r  r  )rv   r   r   r}   r'  Oner,   rQ   rU   r   r   guard_equals)r  r  r   rn   rn   ro   r  ?  s   zView.resolve_negative_sizec              	   C  sX   z	|  ||}W |S  ttfy+   t|g}|  ||}|  ||}t||}Y |S w r   )_dynamic_reshape_indexerr  
IndexErrorrQ   r   )r#  r  r  r   Zflatr   r   rn   rn   ro   r  N  s   
zView.dynamic_reshape_indexerc                   sJ  t jjj}dd tt|D  tt |}t| }g |r|r| }| \}}|dkr>	t
jj |	||f n|dkrH|	| n||||kr^	| t jj|| n||||k r||||k r| \}}	|| | }||	 }||||k sn	| t jj|| nK||||krt
jj}
|}	t||
| |
| }
||||kr| }	t||
| |
| }
|| }||||kst jj|| nt|r|s!|r| }t jj|d 	t
jj |s|r| \}}t jj|d |s  tt| ksJ  fdd}|S )zG
        Perform a reshape entirely by modifying indexing math
        c                 S  r  rn   )rP   r/   ZVIEWr   rn   rn   ro   r   b  r  z1View._dynamic_reshape_indexer.<locals>.<listcomp>r0   c                   sH   t | t ksJ t | t ftt|  t fddD S )Nc                 3  s    | ]}t | V  qd S r   )rR   r  Zreplacementsrn   ro   r    r  zAView._dynamic_reshape_indexer.<locals>.reindex.<locals>.<genexpr>)r   rx   r   rw   r   rF  Z	view_exprr  ro   r     s   $z.View._dynamic_reshape_indexer.<locals>.reindex)rU   r   r   r   r   r   rv   r   r!  r  r}   r'  r(  r  r  r.   r  reverse)r  r  r   Z	stack_newZ	stack_oldZsize_oldvarZsize_newZvar2Z	size_new2Zdivisormodulusr   rn   r  ro   r  Z  sj   



 zView._dynamic_reshape_indexerN)
rj   rk   rl   r  r  r8  r"  r  r  r  rn   rn   rn   ro   r    s    

.

r  c                      s   e Zd ZU dZded< d/ fddZd0d	d
ZeZdd Zd1ddZ	d2ddZ
edd Zd3ddZdd Zd4ddZd5ddZd6d d!Zd"d# Zd7d%d&Zd8d9d*d+Zd:d-d.Z  ZS );r-  z*Pretend our storage has a different layoutrm  r)  rr   rs   c                   s2   t    t| jtrt| d| j  d S d S )Nr(  )r  rE  ru   r(  ry  r   r@  r  rD  r  rn   ro   rE    s   
zReinterpretView.__post_init__r   c                 C     |  | j| jgS r   )re  r(  r)  rD  rn   rn   ro   r    s
   zReinterpretView.__str__c                 C  rz  r   r  rD  rn   rn   ro   r    r  zReinterpretView.get_namer  c                 C     | j jS r   )r)  r   rD  rn   rn   ro   r     ru  zReinterpretView.get_devicer9  c                 C  r   r   rn   rD  rn   rn   ro   rT    r   zReinterpretView.get_origin_nodec                 C  r  r   )r)  r   rD  rn   rn   ro   r     r~  zReinterpretView.dtypery  c                 C     t | jjS r   )rv   r)  r   rD  rn   rn   ro   r     r   zReinterpretView.get_sizec                 C  r  r   )rv   r)  r   rD  rn   rn   ro   r    r   zReinterpretView.get_strider  c                      d fdd}|S )Nr   ry  rr   rT   c                   sF    j  }t  || } j j jjkr!t| j jjS |S r   )r)  r  rS   loadr  r   r(  to_dtype_bitcast)r   rD  Z
tmp_loaderrD  rn   ro   rK    s
   
z+ReinterpretView.make_loader.<locals>.loaderr   ry  rr   rT   rn   r  rn   rD  ro   r    s   zReinterpretView.make_loaderr  c                 C  rz  r   )r)  r  rD  rn   rn   ro   r    r  zReinterpretView.make_indexerc                 C  rP  r   r)  rD  rn   rn   ro   r     rR  zReinterpretView.get_layoutc                 C  r   r   rn   rD  rn   rn   ro   r    r   zReinterpretView.freeze_layoutr   c                 C  s$   t | jjt | jjB t | jjB S r   )r%   r)  r   r   r,  rD  rn   rn   ro   r    s   


z(ReinterpretView.get_unbacked_symbol_usesNr  r  c                 C  s@   t jjj| j| jj| jj| jj|d ur|j	nt jjj	| jj
dS rB  )rU   r   wrapper_codeZcodegen_reinterpret_viewr(  r)  r   r   r,  	writeliner   r  rn   rn   ro   r    s   z!ReinterpretView.codegen_referencer   c                 C  r  r  rn   rD  rn   rn   ro   r    r   zReinterpretView.num_readsr  r  r  r  r  r  r  r  r  r   r  r  )rj   rk   rl   __doc__rm   rE  r  r7  r  r   rT  r  r   r   r  r  r  r   r  r  r  r  r:  rn   rn   r  ro   r-    s(   
 








r-  c                   @  sT   e Zd ZU dZded< edd Zddd	ZeZe	d
d Z
dddZdddZdS )	DtypeViewz(Pretend our storage has a different typerf  target_dtypec                 C  sD   t |rt|\}}t|j||j|j|j}t||dS t||dS )Nr'  )r(  r  )	r   r*  r+  r   r   r   r,  r-  r  )r#  r   Z	new_dtyper.  r/  r1  rn   rn   ro   r"    s   zDtypeView.createrr   r   c                 C  r  r   )re  r(  r  rD  rn   rn   ro   r    r  zDtypeView.__str__c                 C  rP  r   )r  rD  rn   rn   ro   r     s   zDtypeView.dtypery  c                 C  rz  r   r(  r   rD  rn   rn   ro   r     r  zDtypeView.get_sizer  c                   s   j    fdd}|S )Nc                   s   t  | jjjS r   )rS   r  r  r(  r   r%  r  rA  rn   ro   rK    s   z%DtypeView.make_loader.<locals>.loaderr(  r  r  rn   r  ro   r    s   
zDtypeView.make_loaderNr  r  r  )rj   rk   rl   r  rm   r8  r"  r  r7  r  r   r   r  rn   rn   rn   ro   r    s   
 



r  c                   @  s&   e Zd Zedd ZedddZdS )		SliceViewc                   s   t jj| | tdd ||fD rtjtjnjj	fdd  fdd}||dd}|||}||fS )zz
        Normalize start and end such that both are in the range
        [0, x.get_size()[dim]] and start <= end.
        c                 s  r
  r   r  r  rn   rn   ro   r  %  r  z0SliceView.normalize_start_end.<locals>.<genexpr>c                   s<    | |r| n | |}||r|}|S ||}|S r   )statically_known_geqr  )r   lowerupperZclamped_lowerZclamped_full)max_funcmin_funcr   rn   ro   clamp,  s   
z,SliceView.normalize_start_end.<locals>.clampc                   s$   | d u r|S  | }  | ||S r   )r  )r  r  r  r  )r  r#  dim_sizern   ro   
clamp_wrap7  s   z1SliceView.normalize_start_end.<locals>.clamp_wrapr   )
rU   r   r   r   r]  r}   ZMinZMaxZevaluate_minZevaluate_max)r#  r   r  startendr  rn   )r  r#  r  r  r  r   ro   normalize_start_end  s   zSliceView.normalize_start_endr0   Tc                   s  t tt jsdksJ zdkr!|dkr!dkr!|W S W n	 ty+   Y nw t| |r>| | |\}t| d   < t	|rzt
|\}}t|j}	|	   |	 < t|j|j|	|j|j    }
t||
dS  fdd}t||dS )Nr   l    r0   r'  c                   sD   t | t ksJ d|  d t| } |     |  < | S )Nzwrong ndim r  )r   rv   r   r  r  r  steprn   ro   r   c  s   $z!SliceView.create.<locals>.reindexr  )r}   r  ru   r   	TypeErrorrv   r   r  r-   r   r*  r   r+  r   r   r,  r-  r  )r#  r   r  r  r  r  r  r.  r/  r0  r1  r   rn   r  ro   r"  A  s6   

zSliceView.createN)r0   T)rj   rk   rl   r8  r  r"  rn   rn   rn   ro   r    s
    
$r  c                   @  sF   e Zd ZU ded< ded< dddZdd
dZdddZdddZdS )BaseConstantrf  r   r  r   rr   ry  c                 C  r  Nrn   rn   rD  rn   rn   ro   r   r  r   zBaseConstant.get_sizer  c                 C  rP  r   r  rD  rn   rn   ro   r   u  rR  zBaseConstant.get_devicer9  c                 C  r   r   rn   rD  rn   rn   ro   rT  x  r   zBaseConstant.get_origin_noder  c                 C  r  r   r*   rD  rn   rn   ro   rM  {  rR  zBaseConstant.get_readsNr  r  r  r  )rj   rk   rl   rm   r   r   rT  rM  rn   rn   rn   ro   r  m  s   
 


r  c                   @  sD   e Zd ZU ded< ded< ded< dd	d
ZdddZdddZdS )Constantr
   r   rf  r   r  r   rr   r  c                   r  )Nr   ry  rr   rT   c                      t  j jS r   )rS   r>  r   r   r   rD  rn   ro   rK    r  z$Constant.make_loader.<locals>.loaderr  rn   r  rn   rD  ro   r       zConstant.make_loaderr   c                 C  r   r   rn   rD  rn   rn   ro   r    r   zConstant.realizerb   c                 C     t | j| j|dS )N)r   r   r   )r  r   r   r  rn   rn   ro   r    r  zConstant.constant_to_deviceNr  r  r  )rj   rk   rl   rm   r  r  r  rn   rn   rn   ro   r    s   
 

r  c                   @  s:   e Zd ZU ded< ded< ded< dd	d
ZdddZdS )IndexingConstantr
   r   rf  r   r  r   rr   r  c                   r  )Nr   ry  rr   rT   c                   r  r   )rS   r  r   r   r   rD  rn   ro   rK    r  z,IndexingConstant.make_loader.<locals>.loaderr  rn   r  rn   rD  ro   r    r  zIndexingConstant.make_loaderrb   c                 C  r  )N)r   r   r   )r  r   r   r  rn   rn   ro   r    r  z#IndexingConstant.constant_to_deviceNr  r  )rj   rk   rl   rm   r  r  rn   rn   rn   ro   r    s   
 
r  r   c                 C  s    t dd t| t||D S )Nc                 s  s&    | ]\}}}|d kp||kV  qdS r  rn   )r   leftrightr   rn   rn   ro   r    s
    
z2is_contiguous_strides_for_shape.<locals>.<genexpr>)r  r   r   r   )r   r  rn   rn   ro   is_contiguous_strides_for_shape  s
   r  c                 C  s   t j| j S r   )r1   Zpadding_alignment_bytesitemsizerg  rn   rn   ro   get_align_for_dtype  r   r  c                   @  s$   e Zd ZdZd
ddZdddZd	S )r   zxAbstract base for Layout, MultiOutputLayout, NoneLayout.
    Represents the memory layout of the output of an Operation.rr   r  c                 C  r  r   r  rD  rn   rn   ro   r     r  zOutputSpec.get_devicer   c                 C  r  r   r  rD  rn   rn   ro   storage_size  r  zOutputSpec.storage_sizeNr  r  )rj   rk   rl   r  r   r  rn   rn   rn   ro   r     s    
r   c                   @  s   e Zd Zdedfd4ddZd5ddZeZd6ddZd7ddZe	d8ddZ
d7ddZd7d d!Zd"d# Ze	d$d% Zd&d' Zd(d) Zd*d+ Zd9d-d.Zd7d/d0Zd:d2d3ZdS );rm  Nr   r   r  r   rf  r   r  r   Optional[list[Expr]]r,  r   rr   rs   c                 C  sn   |d u r	t |}|| _|| _t|t|ks!J d| d| tdd |D s,J || _|| _|| _d S )Nr  	, stride=c                 s  s    | ]
}t |ttfV  qd S r   )ru   r   r   r   rn   rn   ro   r    s    z"Layout.__init__.<locals>.<genexpr>)	r   r   r   r   r   r  r   r   r,  )rA  r   r   r   r   r,  rn   rn   ro   r    s   
$
zLayout.__init__r   c                 C  sr   d}| j dkrd| j  }| jjd u rdnd| jj }t| j d| jj | d| j d| j d| j | d	S )
NrY  r   z	, offset=:z('z', z, size=r  r   )r,  r   r   r   rj   r   r   r   )rA  r,  Zdevice_index_strrn   rn   ro   r    s   
"zLayout.__str__c                 C  rP  r   r  rD  rn   rn   ro   r     rR  zLayout.get_devicerh   c                 C  s   t | j| jS r   )r  r   r   rD  rn   rn   ro   rk    r  zLayout.is_contiguousr  r  r!  c                 C  sV   t | }|dvs| d dkrdS t|t| | D ]\}}}|dkr(||kr( dS qdS )N)r      r0   FT)r   r   r!   )r  r!  ndimr  r  r   rn   rn   ro   is_channels_last_contiguous  s   
z"Layout.is_channels_last_contiguousc                 C  sJ   t | jtttt| j| jD ]\}}}|dkr"||kr" dS qdS )Nr0   FT)r   r   reversedr   r   rv   r   )rA  r  r  r   rn   rn   ro   is_transposed  s   zLayout.is_transposedc                   s   t jt  ksJ dd tjD }fdd|D } fdd|D  dd }|  dgt   }tt  D ]
}|| | | < q<tt  d D ]'}|| ||d  k}t|tsqtjj	j
|| ||d  kd	d
}|rv dS qOd	S )Nc                 S  s*   g | ]\}}t jjj|d ddkr|qS )r   r  r0   rU   r   r   r   )r   r   r  rn   rn   ro   r     s
    z,Layout.is_stride_ordered.<locals>.<listcomp>c                   r  rn   r  r   rD  rn   ro   r     r   c                   r   rn   rn   r   r   rn   ro   r     r   c                   s   t |   fdd| D S )Nc                      g | ]}  |qS rn   r   )r   elementZ
sorted_arrrn   ro   r   
  r   zDLayout.is_stride_ordered.<locals>.sorted_indices.<locals>.<listcomp>)r  )Zarrrn   r  ro   sorted_indices     z0Layout.is_stride_ordered.<locals>.sorted_indicesr  r0   Tr  F)r   r   r   r   r   ru   rh   rU   r   Z
_shape_envr  )rA  r   Znon_1_indicesr   r  stride_orderedr   exprrn   )r   rA  ro   rw    s*   
zLayout.is_stride_orderedc                 C  s:   dgt ttdt| jd  }t|g| }| |S Nr   r0   )rv   r  r   r   r   rw  r  rn   rn   ro   is_channels_last_stride_ordered  s   "
z&Layout.is_channels_last_stride_orderedc                 C  s*  t |}t| dkr| S tjst|| r| S t }t|dr)|j	
ddr)| S tdd t| |D s8| S t| }t|}dd tt| D }d	||d < d}t|d	d
 d	dD ]*\}	}
||	d	  }|| ||  }|tjkr|| dkrt||| }d}|||
< q]|s| S t jd	7  _|S )z
        The padding does not change stride order but makes sure all strides larger
        than the threshold are multiple of align.
        r   metaZdislike_paddingFc                 s  s     | ]}t |ttjfV  qd S r   )ru   r   r}   r   r   rn   rn   ro   r  >  r0  z&Layout._pad_strides.<locals>.<genexpr>c                 S  r   r   rn   r   rn   rn   ro   r   G  r   z'Layout._pad_strides.<locals>.<listcomp>r0   N)r  T)r  r   r1   Zpad_channels_lastrm  r  rU   Zget_current_noder  r
  getr  r  chainr   r   r   r   Zpadding_stride_thresholdrF   r   Znum_comprehensive_padding)Z
in_stridesr   r   alignZcurrent_fx_nodero  r   Znew_stridespaddedrankr   Zprev_idxr   rn   rn   ro   _pad_strides$  s@   


zLayout._pad_stridesc                 C  s6   t | tsJ | jd usJ | | j| j| j| _d S r   )ru   r   r   r  r   r   rD  rn   rn   ro   rj  _  s   zLayout.pad_stridesc                 C  s   t jot| tS r   )r1   comprehensive_paddingru   r   rD  rn   rn   ro   ri  d  r  zLayout.should_pad_stridesc                 C  s8   t | tr| S |  r|   t| j| j| j| j| jS r   )	ru   r+  ri  rj  r   r   r   r   r,  rD  rn   rn   ro   as_fixedg  s   
zLayout.as_fixedr  c                 C  s(   t jsJ dt| j d|   S )Nzconvert z to FixedLayout first)r   r-  r   rj   r  r  rD  rn   rn   ro   r  u  s   zLayout.make_indexerc                 C  s<   | j |j ko| j|jko| j|jko| j|jko| j|jkS r   r   r   r   r   r,  )rA  otherrn   rn   ro   __eq__{  s   



zLayout.__eq__
sympy.Exprc                 C  s   t | j| j| jS r   )r   r   r   r,  rD  rn   rn   ro   r    r  zLayout.storage_size)r   r  r   rf  r   r  r   r  r,  r   rr   rs   r  r  r  )r  r  r!  r  rr   rh   r  rr   r  )rj   rk   rl   r   r  r  r7  r   rk  r  r  r  rw  r	  r  rj  ri  r  r  r  r  rn   rn   rn   ro   rm    s*    





#
:

	rm  c                   @  s   e Zd ZdZdddZdS )r+  z A Tensor layout we cannot changerr   r  c                   s    fdd}|S )z1A closure containing math to read a given elementc                   sf   t | t  jksJ t | t  jksJ  j}t|  j jD ]\}}}|dkr0|||  }q!|S r  )r   r   r   r,  r   )r   r>  r   r   szrD  rn   ro   rD    s   z)FixedLayout.make_indexer.<locals>.indexerrn   r  rn   rD  ro   r    s   	zFixedLayout.make_indexerNr  )rj   rk   rl   r  r  rn   rn   rn   ro   r+    s    r+  c                      s   e Zd ZdZdZedd Zedd Zedd Zed	d
 Z	edd Z
dddZdddZdd Zdd Zdd fddZ  ZS )r   z(A Tensor layout we are allowed to changeFc                 C  sN   t | dkrg S tjjg}t| dd  D ]}|||d   qtt|S )Nr   r0   r  )r   r}   r'  r  r  r  rv   )sizesZreversed_stridesr   rn   rn   ro   r     s   
z!FlexibleLayout.contiguous_stridesc                 C  s\   t tt| t |ksJ | |ftjj}dgt| }|D ]}|||< || |  }q|S )z
        Create a stride based on the order the dimensions should be filled in.

        In this format, channels last would be:
            [1, 3, 2, 0]
        N)r+   r   r   r}   r'  r  )r  r   Znext_strider!  r   rn   rn   ro   fill_ordered  s   $zFlexibleLayout.fill_orderedc                 C  s0   t tt| t |ksJ t|}t| |S )z
        Create a stride based on the sorted order of a permuted range.

        In this format, channels last would be:
            [3, 0, 2, 1]
        )r+   r   r   r   r   r  )r  r   r   rn   rn   ro   r    s   zFlexibleLayout.stride_orderedc                 C  sP   |t jkrt| tS |t jkrt| tS |t jkr t| S t	
d| t)aq  
        Create a stride based on a memory format.

        Memory format is translasted into a stride order,
        so channels_last is the same as:
            FlexibleLayout.stride_ordered(sizes, [3, 0, 2, 1])

        This interface does not support memory_format `torch.preserve_format`
        which should be used to deduce a format from another source
        z>stride_ordered_for_memory_format, unsuppored memory_format: %s)r   channels_lastr   r  NHWC_STRIDE_ORDERchannels_last_3dNHWDC_STRIDE_ORDERZcontiguous_formatr   r  r  rj  )r  memory_formatrn   rn   ro    stride_ordered_for_memory_format  s   



z/FlexibleLayout.stride_ordered_for_memory_formatc                 C  sD   t | t |ks
J dd |D }ttt ||jd}t| |S )z
        Create a stride that has the same stride order as given stride

        For example, if given stride is [1000, 1, 100, 10],
        the fill order should be [1, 3, 2, 0]
        c                 S  r  rn   r   r  rn   rn   ro   r     r  z/FlexibleLayout.same_ordered.<locals>.<listcomp>r  )r   r  r   __getitem__r   r  )r  r   r   rn   rn   ro   same_ordered  s   zFlexibleLayout.same_orderedc                 C  sD   |  | j|}|  r|r| || j| j}t| j| j| j|| jS r   )r  r   ri  r  r   r+  r   r,  )rA  r   r  r0  rn   rn   ro   as_stride_order  s   zFlexibleLayout.as_stride_orderc                 C  s:   |}|   r|r| || j| j}t| j| j| j|| jS r   )ri  r  r   r   r+  r   r,  )rA  r  r  r0  rn   rn   ro   as_exact_strides  s   zFlexibleLayout.as_exact_stridesc                 C  @   |  | j|}|  r| || j| j}t| j| j| j|| jS r   )r  r   ri  r  r   r+  r   r,  )rA  r   r0  rn   rn   ro   as_fill_order     zFlexibleLayout.as_fill_orderc                 C  r%  r   )r"  r   ri  r  r   r+  r   r,  )rA  r   r0  rn   rn   ro   as_same_order  r'  zFlexibleLayout.as_same_orderNrr   rs   c                   s2   |r	t ||}nt |}t |||| d S r   )r   r  r   r  r  )rA  r   r   r   ro  r!  r  rn   ro   r    s   
zFlexibleLayout.__init__r  r   r  )rj   rk   rl   r  r-  r  r   r  r  r   r"  r#  r$  r&  r(  r  r:  rn   rn   r  ro   r     s$    






r   c                      s4   e Zd ZdZd fddZdd	d
Zdd Z  ZS )NonOwningLayoutz,Is a view into the storage of another tensorviewUnion[BaseView, TensorBox]rr   rs   c                   s,   |  }t |j|j|j|j || _d S r   )r   r  r  r   r   r   r   r*  )rA  r*  r)  r  rn   ro   r  )  s   
zNonOwningLayout.__init__r  c                 C     |    S r   )r  r  rD  rn   rn   ro   r  3  r   zNonOwningLayout.make_indexerc                 C  s4   | j  j}|dkrdS ddlm} tjj||S )Nr   Tr0   )	ALIGNMENT)	r*  r   r,  utilsr-  rU   r   r   Zstatically_known_multiple_of)rA  r,  r-  rn   rn   ro   maybe_guard_aligned6  s
   z#NonOwningLayout.maybe_guard_aligned)r*  r+  rr   rs   r  )rj   rk   rl   r  r  r  r/  r:  rn   rn   r  ro   r)  &  s
    

r)  c                   @     e Zd ZdZdS )CommBufferTypeZsymm_memN)rj   rk   rl   ZSYMM_MEMrn   rn   rn   ro   r1  ?      r1  c                      s4   e Zd ZU dZded< ded< d
 fdd	Z  ZS )CommBufferLayoutax  
    A layout that signifies the buffer is a comm buffer.
    In terms of striding, the layout is identical to `FixedLayout`.

    Buffers with this layout do not participate in in-place reuse - it can be
    neither the source nor the target for in-place reuse.

    For detailed motivation and usage of this layout, see
    NOTE [lowering-time collective optimization].
    r1  comm_buffer_typer   
group_namer)  r   c                   sR   t |tstd| d| }t j|j|j|j|j	|j
d || _|| _d S )NzJA `CommBufferLayout` can only be initialized with a `FlexibleLayout` (got z).r  )ru   r   r  r  r  r  r   r   r   r   r,  r4  r5  )rA  r)  r4  r5  fixedr  rn   ro   r  R  s    

zCommBufferLayout.__init__)r)  r   r4  r1  r5  r   )rj   rk   rl   r  rm   r  r:  rn   rn   r  ro   r3  C  s
   
 r3  c                   @  sb   e Zd ZU ded< ejdd dZded< ejdd dZded	< dddZdd Z	dddZ
dS )
NoneLayoutr  r   c                   C     dgS r  rn   rn   rn   rn   ro   r  u  r  zNoneLayout.<lambda>default_factoryr  r   c                   C  r8  r  rn   rn   rn   rn   ro   r  v  r  r   rr   r   c                 C  r  r  rn   rD  rn   rn   ro   r  x  r   zNoneLayout.storage_sizec                 C     | S r   rn   rD  rn   rn   ro   r  {  r   zNoneLayout.as_fixedc                 C  rP  r   r  rD  rn   rn   ro   r   ~  rR  zNoneLayout.get_deviceNr  r  )rj   rk   rl   rm   r  r  r   r   r  r  r   rn   rn   rn   ro   r7  j  s   
 

r7  c                      sx   e Zd Zd fddZeddd	Zejddd	Zd ddZd!ddZdd Z	e
d"ddZdd Zd#ddZ  ZS )$MutationLayoutSHOULDREMOVEr  rb   rr   rs   c                   s@   t  | | | d  || _|   }tj	
| d S r   )r  r  r  r   r   r  
get_bufferr  rU   r   mark_buffer_mutated)rA  r  r   r  rn   ro   r    s   z#MutationLayoutSHOULDREMOVE.__init__r  c                 C  r  r   )real_layoutr   rD  rn   rn   ro   r     r  z!MutationLayoutSHOULDREMOVE.strider   r   c                 C  r   r   rn   )rA  r   rn   rn   ro   r     r?  r  c                 C  r,  r   )r?  r  rD  rn   rn   ro   r    r   z'MutationLayoutSHOULDREMOVE.storage_sizeru  c                   s,    fdd  | j }t|tsJ d|S )Nc                   sB   t | tr
 | jS t | tr |  S t | tr | jS | S r   )ru   r<  r  ry  r  
MutableBoxr(  )r  unwrap_viewsrn   ro   rB    s   




z;MutationLayoutSHOULDREMOVE.get_buffer.<locals>.unwrap_viewsz1MutationLayoutSHOULDREMOVE must refer to a buffer)r  ru   ru  )rA  r>  rn   rA  ro   r=    s   
	z%MutationLayoutSHOULDREMOVE.get_bufferc                 C  r  r   )r=  r)  rD  rn   rn   ro   r?    r  z&MutationLayoutSHOULDREMOVE.real_layoutFc              	   C  s   |   tj|  t|tr|j}|  |s6t	j
| | | dd t| | D dj}|   t|jjtsCJ t||j_|jS )Nc                 S      g | ]\}}t jj||qS rn   rU   r   r   r  r   ri  rk  rn   rn   ro   r         z;MutationLayoutSHOULDREMOVE.realize_into.<locals>.<listcomp>rP  )r  rU   r   r>  r  ru   ra   r(  r  rA  r"  r   r   r  r   r   r)  r   r<  )r#  srcdstZunsafe_aliasrn   rn   ro   realize_into  s(   

z'MutationLayoutSHOULDREMOVE.realize_intoc                 C  r;  r   rn   rD  rn   rn   ro   r    r   z#MutationLayoutSHOULDREMOVE.as_fixedr  c                 C  rz  r   )r  r  rD  rn   rn   ro   r    r  z'MutationLayoutSHOULDREMOVE.make_indexer)r  rb   rr   rs   rr   r  )r   r   rr   rs   r  )rr   ru  r  r  )rj   rk   rl   r  r  r   setterr  r=  r?  r8  rI  r  r  r:  rn   rn   r  ro   r<    s    

"r<  c                      sB  e Zd ZU ded< ded< dQ fddZdRd
dZdSddZdTddZdUddZe	dVddZ
dWddZdXddZdYdd ZdZd"d#Zd[d$d%Zd&d' Zd(d) Zd\dQd+d,ZdQd-d.ZdQd/d0Z	*d\dQd1d2Zd3d4 Zd]d6d7Zd^d_d;d<Zd=d> Zd`d@dAZd`dBdCZdadEdFZdbdHdIZdbdJdKZdcdLdMZdddOdPZ   Z!S )eru  r   r   r   r)  rr   rs   c                   s   t    | dd  d S r  )r  rE  rB  rD  r  rn   ro   rE    s   
zBuffer.__post_init__r  c                 C  r,  r   )r   r  rD  rn   rn   ro   r    r   zBuffer.make_indexerr   c                 C  s   | j sJ | | j S r   r   rD  rn   rn   ro   r    r  zBuffer.get_namer  c                 C  r,  r   )rt  r   rD  rn   rn   ro   r     r   zBuffer.get_devicerU  c                 C  r   r   rn   rD  rn   rn   ro   rV    r   zBuffer.get_defining_oprf  c                 C  r  r   )r   r   rD  rn   rn   ro   r     r  zBuffer.dtypery  c                 C     g |   jS r   )r   r   rD  rn   rn   ro   r     r  zBuffer.get_sizer  c                 C  rL  r   )r   r   rD  rn   rn   ro   r    r  zBuffer.get_strider   c                 C  r  r   )r   r,  rD  rn   rn   ro   
get_offset  r  zBuffer.get_offsetrm  c                 C  s"   t | jtr	| jS tt| jjr   )ru   r)  rm  rj  r   rj   rD  rn   rn   ro   r     s   zBuffer.get_layoutc                 C  rP  r   r  rD  rn   rn   ro   rt    rR  zBuffer.get_output_specc                 C  rs  r   )r  rD  rn   rn   ro   r    ru  zBuffer.get_storage_numelc                 C  s0   t | jtrt | jts| j | _d S d S d S r   )ru   r)  rm  r)  r  rD  rn   rn   ro   r  	  s
   zBuffer.freeze_layoutFc                 C  &   t | jtsJ | jj||d| _d S Nrs  )ru   r)  r   r#  r  rn   rn   ro   r    s   z&Buffer.freeze_layout_with_stride_orderc                 C  "   t | jtsJ | j|| _d S r   )ru   r)  r   r&  r  rn   rn   ro   r       z$Buffer.freeze_layout_with_fill_orderc                 C  rP  r   )ru   r)  r   r(  r  rn   rn   ro   r    rQ  z$Buffer.freeze_layout_with_same_orderc                 C  rN  rO  )ru   r)  r   r$  r  rn   rn   ro   r    s   z'Buffer.freeze_layout_with_exact_stridesc                 C  r  r  r  rD  rn   rn   ro   r  #  r  zBuffer.is_zero_elementsr  c                   s(      rtt  dS  fdd}|S )Nrg  c                   s      }t jp
d|| S rG  )r  rS   r  r   r   rD  rD  rn   ro   rK  +  s   z"Buffer.make_loader.<locals>.loader)r  r	   r@  r   r  rn   rD  ro   r  &  s   zBuffer.make_loaderNr  r  c                 C  rs  r   r  r  rn   rn   ro   r  1  ru  zBuffer.codegen_referencec                 C  r   r   rn   rD  rn   rn   ro   r  4  r   zBuffer.decide_layoutrX  c                 C     t | jtr| jj gS dS r  )ru   r)  r)  r*  r  rD  rn   rn   ro   r  7     z#Buffer.get_inputs_that_alias_outputc                 C  rT  r  )ru   r)  r<  r  r  rD  rn   rn   ro   r  <  rU  zBuffer.get_mutation_namesrF  c                 C  s   t |  gS r   )r+   r  rD  rn   rn   ro   rN  A  r  zBuffer.get_read_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r  D  rR  zBuffer.get_unbacked_symbol_usesc                 C  r  r   r*   rD  rn   rn   ro   r  G  rR  zBuffer.get_unbacked_symbol_defsc                 C  r   r   rn   rD  rn   rn   ro   r  J  r   zBuffer.realizerh   c                 C  r  r  rn   rD  rn   rn   ro   should_allocateM  r?  zBuffer.should_allocater  r  r  r  r  r  r  rJ  r  r  r  r  r  r   r  r  r  r  r  r  )"rj   rk   rl   rm   rE  r  r  r   rV  r  r   r   r  rM  r   rt  r  r  r  r  r  r  r  r  r  r  r  r  rN  r  r  r  rV  r:  rn   rn   r  ro   ru    sB   
 

















ru  c                   @  s0   e Zd ZdddZdddZejZdd	d
ZdS )OperationBufferrr   r  c                 C  s   | gS r   rn   rD  rn   rn   ro   r  U  rR  zOperationBuffer.get_outputsr  c                 C  r;  r   rn   rD  rn   rn   ro   rV  X  r   zOperationBuffer.get_defining_oprs   c                 C  s   t |  t|  d S r   )ru  rE  r  rD  rn   rn   ro   rE  ^  s   
zOperationBuffer.__post_init__Nr  rr   r  r  )rj   rk   rl   r  rV  r  r  rE  rn   rn   rn   ro   rW  R  s
    

rW  c                   @     e Zd ZdddZdS )InputBufferrr   r   c                 C  r  r  rn   rD  rn   rn   ro   r  d  r   zInputBuffer.num_readsNr  )rj   rk   rl   r  rn   rn   rn   ro   rZ  c      rZ  c                   @  r0  )DonatedBufferaY  
    Represents a donated buffer which is a saved tensor that is not alias to any
    fwd inputs, fwd user outputs, and bwd outputs. We generally cannot inplace
    reuse the input tensor memory during backward since it might be used in another
    function. However, donated buffer can be inplace reused during backward
    to save memory.
    N)rj   rk   rl   r  rn   rn   rn   ro   r\  h  r2  r\  c                   @  s.   e Zd ZU dZded< dddZdddZdS )rQ  Nr  rO  rr   r  c                   r  )Nr   ry  rr   rT   c                   s,       }ttj   j|| S r   )	r   r  rS   r  rU   r   constant_namer  rO  rR  rD  rn   ro   rK  v  s
   z*ConstantBuffer.make_loader.<locals>.loaderr  rn   r  rn   rD  ro   r  u  s   zConstantBuffer.make_loaderr   r  rb   c                 C  s   t tj|  || jdS N)r   r)  )rQ  rU   r   r]  r  r)  r  rn   rn   ro   r    s   z!ConstantBuffer.constant_to_devicer  r  )rj   rk   rl   rO  rm   r  r  rn   rn   rn   ro   rQ  r  s   
 

rQ  c                   @  s@   e Zd ZdddZdddZddddZdddZdddZdS )NoneAsConstantBufferrr   r  c                 C  r  r   r*   rD  rn   rn   ro   rM    rR  zNoneAsConstantBuffer.get_readsr   c                 C  r  r   r*   rD  rn   rn   ro   r    rR  z-NoneAsConstantBuffer.get_unbacked_symbol_usesNr  r  r   c                 C  s
   t jjjS r   )rU   r   r  none_strr  rn   rn   ro   r    r  z&NoneAsConstantBuffer.codegen_referencer   c                 C  s
   t d dS Nr  )r7  rD  rn   rn   ro   rt    r  z$NoneAsConstantBuffer.get_output_specrh   c                 C  r  r  rn   rD  rn   rn   ro   rx    r   z&NoneAsConstantBuffer.has_tensor_outputr  r  r   r  r  r  )rj   rk   rl   rM  r  r  rt  rx  rn   rn   rn   ro   r_    s    


r_  c                   @  s6   e Zd ZU ded< dddZddddZdddZdS )r   r   r  rr   r   c                 C  
   t | jS r   )r%   r  rD  rn   rn   ro   r    r  z.ShapeAsConstantBuffer.get_unbacked_symbol_usesNr  r  r   c                 C  s   t jj| jS r   )rU   r   r  Zcodegen_sizevarr  r  rn   rn   ro   r    r  z'ShapeAsConstantBuffer.codegen_referencerh   c                 C  r  r  rn   rD  rn   rn   ro   rx    r   z'ShapeAsConstantBuffer.has_tensor_outputr  r   r  r  )rj   rk   rl   rm   r  r  rx  rn   rn   rn   ro   r     s
   
 
r   c                      s   e Zd ZU ded< d>ddZd?dd	Zd@ddZdAddZdBddZdCddZ	dD fddZ
dEddZdFddZdGd d!ZedHd#d$Z	%	%dIdJd+d,Ze	%dKd-d.ZdLd0d1Zd>d2d3ZdMd5d6ZdMd7d8ZdNd<d=Z  ZS )Or  r  r(  rr   r   c                 C  s(   | j dur| j S t| jdr| jj S dS )z
        Returns self.name if it exists, otherwise returns the name of the data node if that exists.
        If neither exist, returns None.
        Nr   )r   r  r(  rD  rn   rn   ro   get_computed_buffer_name  s
   
z'ComputedBuffer.get_computed_buffer_namer   c                 C  rz  r   r(  r  rD  rn   rn   ro   r    r  zComputedBuffer.num_readsr  c                 C  rz  r   r(  rM  rD  rn   rn   ro   rM    r  zComputedBuffer.get_readsrF  c                 C  rz  r   r  rD  rn   rn   ro   rN    r  zComputedBuffer.get_read_namesr  c                 C  sz   t tdd, | j r"t|  | j | j W  d    S t|  | j	 W  d    S 1 s6w   Y  d S r,  )
r   r   r   r(  r  r9   get_store_functionr   r  r   rD  rn   rn   ro   r    s   
$zComputedBuffer.get_read_writesr   c                 C  s.   t |  t |  B t |  B | j B S r   )r%   r   r  rM  r(  r  rD  rn   rn   ro   r    s   


z'ComputedBuffer.get_unbacked_symbol_usesr  c                   s6   |   s| jtjjvr|  dkr| j S t  S r  )	r  r   rU   r   Zmutated_buffersr  r(  r  r  rD  r  rn   ro   r    s   

zComputedBuffer.make_loaderCallable[..., None]c                 C  sV   |     }t| jtttfrt| jj	| j
|S t| jts"J t| jj| j
|S r   )r   r  r  ru   r(  r|  r1  r_  r	   r  r   rA  rL  r  rn   rn   ro   rf    s
   z!ComputedBuffer.get_store_functionOptional[list[int]]c                   s   t | jtrYt| j | j \\}}|  j	}t
dd |D s&J fdd|D }|rYt | jttfrA| j| n|  fdd|D }ddlm} |||  S dS )	al  
        If our layout is still flexible, try to determine the stride order based on stride orders of reads.

        TODO(jansel): A better algorithm here would look at downstream consumers of this
                      value and try to do global graph-level layout optimization.
                      This is also something just begging to be autotuned.
        c                 s  s"    | ]}t |tjtjfV  qd S r   )ru   r2   StarDep	MemoryDepr  rn   rn   ro   r    s
    
z0ComputedBuffer.get_fill_order.<locals>.<genexpr>c                   s.   g | ]}t |tjrt|jd d  D qS )c                 S  s   i | ]}|d kr|t jjqS r   r  )r   vrn   rn   ro   r   
      z<ComputedBuffer.get_fill_order.<locals>.<listcomp>.<dictcomp>)ru   r2   rj  rR   r   r  )r  rn   ro   r   	  s    
z1ComputedBuffer.get_fill_order.<locals>.<listcomp>c                   s   g | ]
}t jj| qS rn   rU   r   r   r  r   r  )r  rn   ro   r     s    r0   pick_loop_orderN)ru   r)  r   r2   r  r(  r   r  r  r  r  r1  r_  r   	schedulerrp  r   )rA  
index_varsr   r  Zstride_lengthsrp  rn   )r  r  ro   r     s*   


zComputedBuffer.get_fill_orderrs   c                 C  s6   t | jtr|  }|r| | d S |   d S d S r   )ru   r)  r   r   r  r  r  rn   rn   ro   r    s   zComputedBuffer.decide_layoutetuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody, tuple[list[sympy.Expr], list[sympy.Expr]]]c           
      C  s   t j| j | j dd\}}ttd|   t	| 
 |  r$|n|d d |g|R  }W d    n1 s:w   Y  g }g }g }g }| D ]+\}}	||d v rd|rYJ || ||	 qK||d v slJ || ||	 qK||f|||ffS )Nqr_   rO  r0   r   )r2   r  r(  r   r  r   r   rQ  r   r;   rf  r  itemsr  )
rA  r   
var_rangesr  rr  reduce_vars
index_sizereduce_sizerk  r   rn   rn   ro   get_default_sizes_body%  s2   



z%ComputedBuffer.get_default_sizes_bodyNextra_indexing_constraints*Optional[tuple[dict[Any, Any], list[Any]]]recompute_sizes_body_funcOptional[Callable[..., Any]]:tuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody]c                   s    \\}}}\}}|r|||f|||f\\}}}\}}g |j  |durmt|tr4t|dks6J |\}}	t|tsAJ t|	tsHJ tdd |	D sSJ |j	}
|
|ks`J |
|f fdd|	D }	 |	7  g |
 tjtjs|   fdd}|| }tt ptj }|||||\}}}|||||\}}}tj||d	d
\\}}}t|||||g|||}||f|fS )an  
        This is a main place where we do loop transformations in a
        backend-agnostic way.

        Here we:
            1) Remove any 1 dimensions
            2) Fuse contiguous dimensions together
            3) Reorder dimensions based on stride orders

        Optional argument extra_indexing_constraints can be used to append additional
        indexing expressions to existing ones derived from buffer's body. This can be useful
        to fuse scheduler nodes with compatible ranges, e.g. (s0*s1*...,) and (s0, s1, s2, ...)
        on CPU by preventing indexing simplifications and obtaining index/reduce ranges for
        the scheduler node compatible with other nodes.
        Optional argument recompute_sizes_body_func can be used to recompute sizes and body
        on the default body. This can be useful to append additional loop transformations.
        Nr   c                 s  s    | ]}t |tV  qd S r   )ru   r   )r   frn   rn   ro   r  t  r  z6ComputedBuffer.simplify_and_reorder.<locals>.<genexpr>c                   s   g | ]}| vr|qS rn   rn   r  )index_formulasrn   ro   r   |      z7ComputedBuffer.simplify_and_reorder.<locals>.<listcomp>c           	        s\    | ||\}}}|| } |r'tjj| |t | |\}}}t||}n|}|||fS r   )_apply_loop_reorderingrU   r   r   _simplify_loopsr5   r   )	Zx_varssupport_varsr  Zsimplify_loopsZreindex0r   r   _pruner   r  memory_addrsrA  rn   ro   simplify_and_reorder  s   



zAComputedBuffer.simplify_and_reorder.<locals>.simplify_and_reorderpr_   )rz  Zindexing_exprsry   ru   rw   r   rx   rv   r  rv  Zget_write_exprsrU   r   r  r3   ZPREFER_STORE_LOOP_ORDERextendZget_read_exprsrM   r  r1   Zloop_ordering_after_fusionr2   Zindex_vars_no_squeezer;   )rA  r{  r}  rx  ry  r  rr  rw  Zextra_indexing_rangesZextra_indexing_exprZexpected_var_rangesr  r  Zshould_merge_loopsZiter_rangesZiter_reindexr   Zreduce_rangesZreduce_reindexZ	iter_varsrv  rn   r  ro   r  F  sx   



z#ComputedBuffer.simplify_and_reorderc              
     s   ddl m} |du rg }z* fdd|D }t|t|kr)t|d t ks+J tt|||}W n  tyV   tjrLt	dt
t | ttt}Y nw fdd|D t|t|fS )	zU
        Shuffle the order of loops around to hopefully improve performance.
        r0   ro  Nc                   s   g | ]}t jj| qS rn   rm  rn  )rr  r  rn   ro   r         z9ComputedBuffer._apply_loop_reordering.<locals>.<listcomp>r   z%Did not simplify complex index:
%s
%sc                   r   rn   rn   r   )r  rn   ro   r     r   )rq  rp  r   rv   r  	Exceptionr1   r  r  warningrx   r   r   r   r   )rr  r  r  r  Zpriority_idxrp  r!  r   rn   )rr  r  r  ro   r    s,   
z%ComputedBuffer._apply_loop_reorderingr  c                 C  rz  r   r(  r  rD  rn   rn   ro   r    r  z!ComputedBuffer.get_reduction_sizec                 C  rz  r   r(  r  rD  rn   rn   ro   r    r  z!ComputedBuffer.get_reduction_typerh   c                 C  rz  r   )r(  r  rD  rn   rn   ro   r    r  zComputedBuffer.is_no_opc                 C  r  NTrn   rD  rn   rn   ro   rV    r   zComputedBuffer.should_allocater   r  rb   c                 C  r  )rN  r(  r  r  rn   rn   ro   r       z!ComputedBuffer.constant_to_devicer  r  r  r  r  r  r  )rr   rg  )rr   rh  r  )rr   rs  NN)r{  r|  r}  r~  rr   r  r   r  r  r  )rj   rk   rl   rm   rc  r  rM  rN  r  r  r  rf  r   r  rE   rz  r  r  r  r  r  r  rV  r  r:  rn   rn   r  ro   r    s2   
 









'"s
#


r  c                      sb   e Zd ZdZd! fd
dZd"ddZdd Zd#ddZd$ddZd%ddZ			d&d'dd Z
  ZS )(TemplateBufferzt
    Represents a Triton (in the future other type) of template operator
    that we can fuse an epilogue onto.
    r)  rm  inputsSequence[IRNode]make_kernel_renderr  rr   rs   c                   s@   t  jd |d t|| _|| _tj| | _	tj
|  d S r^  )r  r  InputsKernelunwrap_storager  r  rU   r   register_bufferr   register_operation)rA  r)  r  r  r  rn   ro   r    s
   zTemplateBuffer.__init__r  c                 C  s   | j ddS )NT	normalize)r9   rD  rn   rn   ro   r     r   zTemplateBuffer.get_read_writesc              	     s   |   |     fdd}tj||  d|d}| jD ]j   fdd}| jtj| dddjO  _q|S )Nc                   s"   t |dksJ t | dS )Nr   Zfake)r   rS   rI  r  )rD  r   rn   ro   dummy  rQ  z1TemplateBuffer.extract_read_writes.<locals>.dummyrn   r  c                   s(   t |dksJ t  |  d S r  )r   rS   r  r  r  )rD  inprn   ro   r    s   T)	r  r   r  r2   r9   r   r  r)  r  )rA  r  r  depsrn   )rD  r  r   ro   r9     s   


z"TemplateBuffer.extract_read_writesr  c                 C  s   t jjS r   )r}   r'  r  rD  rn   rn   ro   r    ru  z!TemplateBuffer.get_reduction_sizer   c                 C  r   r   rn   rD  rn   rn   ro   r    r   z!TemplateBuffer.get_reduction_typerh   c                 C  r  r  rn   rD  rn   rn   ro   rV  "  r   zTemplateBuffer.should_allocateNr{  r|  r}  r~  c                 C  s   |   dfd fS r  r}  )rA  r{  r}  rn   rn   ro   r  %  s
   z#TemplateBuffer.simplify_and_reorder)r)  rm  r  r  r  r  rr   rs   r  r  r  r  r  )r{  r|  r}  r~  )rj   rk   rl   r  r  r  r9   r  r  rV  r  r:  rn   rn   r  ro   r    s    



r  c                      sB   e Zd Z		dd fdd	ZdddZdddZdddZ  ZS )TritonTemplateBufferNmutated_inputsOptional[Iterable[IRNode]]allowed_prologue_inpsOptional[OrderedSet[str]]rr   rs   c                   s   t  ||| |_g_|durFtjjjtjjjf}t	j
jj}||v s0J d| d| jd    j fdd|D 7  _|rM|_dS t _dS )a  
        NOTE:[TritonTemplates with multiple outputs]
        We want the ability for TritonTemplates to output multiple tensors. Triton
        kernels have no notion of outputs and this is done by creating tensors that
        are then mutated by the kernel. Currenlty our STORE_OUTPUT codegen doesn't
        support creating multinode outputs for triton templates.
        We work around this by creating an extra input buffer during the lowering
        and we mark them as mutated inputs.
        Nz$Mutated inputs are only allowed for z	 but got r   c                      g | ]}t t d |qS r  MutationOutputr7  r   r  r   rA  rn   ro   r   S  r  z1TritonTemplateBuffer.__init__.<locals>.<listcomp>)r  r  r  outputsr   rS   Zhigher_orderZflex_attentionZflex_attention_backwardrU   r   current_noder  r  r   r+   r  )rA  r)  r  r  r  r  Zallowed_setr  r  r  ro   r  4  s&   


zTritonTemplateBuffer.__init__r  c                 C  rP  r   )r  rD  rn   rn   ro   r  \  rR  z TritonTemplateBuffer.get_outputsrF  c                 C  rP  r   )r  rD  rn   rn   ro   get_allowed_prologue_inps_  rR  z.TritonTemplateBuffer.get_allowed_prologue_inpsr   c                 C  s   d| j  d}|S )NzTritonTemplateBuffer(layout=r   r  )rA  r   rn   rn   ro   r  b  s   zTritonTemplateBuffer.__str__r  )r  r  r  r  rr   rs   r  r  r  )rj   rk   rl   r  r  r  r  r:  rn   rn   r  ro   r  3  s    
(
r  c                      sf   e Zd ZdZd fddZdddZd ddZdd Zd ddZd!ddZ	d"ddZ
d ddZ  ZS )#ChoiceCallera.  
    Represents a possible choice used in autotune_process.py.
    During autotuning, self.benchmark() is first called to get benchmark result,
    and if this choice is selected, self.output_node() is called to get the output_node.

    Children classes: TritonTemplateCaller, CUDATemplateCaller.
    r   r   rf   r  r)  rm  descriptionrr   rs   c                   s&   t    || _|| _|| _|| _d S r   )r  r  r   r)  rf   r  )rA  r   rf   r)  r  r  rn   ro   r  s  s
   

zChoiceCaller.__init__r?  c                G  s   |   }t||d|iS )Nr   )to_callabler@   	benchmark)rA  r   r   algorn   rn   ro   r    r  zChoiceCaller.benchmarkc                 C  r  r   r  rD  rn   rn   ro   	call_name  r   zChoiceCaller.call_namec                 C  r  r   r  rD  rn   rn   ro   r    r   zChoiceCaller.to_callablec                 C  r  r   r  rD  rn   rn   ro   hash_key  r   zChoiceCaller.hash_keyra   c                 C  r  r   r  rD  rn   rn   ro   output_node  r   zChoiceCaller.output_node<dict[str, Union[PrimitiveInfoType, list[PrimitiveInfoType]]]c                 C  s   i S )zRInformation returned here is logged to the autotune log file when that is enabled.rn   rD  rn   rn   ro   	info_dict  r?  zChoiceCaller.info_dictc                 C  r  )NZunsupported_choicern   rD  rn   rn   ro   autoheuristic_id  r   zChoiceCaller.autoheuristic_id)
r   r   rf   r  r)  rm  r  r   rr   rs   )rr   r?  r  )rr   ra   )rr   r  )rj   rk   rl   r  r  r  r  r  r  r  r  r  r:  rn   rn   r  ro   r  j  s    




r  c                   @  rY  )TritonTemplateCallerBaserr   r
   c                 C  r  r   r  rD  rn   rn   ro   get_make_kernel_render  r   z/TritonTemplateCallerBase.get_make_kernel_renderN)rr   r
   )rj   rk   rl   r  rn   rn   rn   ro   r    r[  r  c                      sb   e Zd ZdZd fddZed ddZed!ddZej	d"ddZ
d#ddZd$ddZ  ZS )%MultiTemplateBufferaG  
    Represents a Buffer with multiple backing implementation choices.

    Choices can be TritonTemplates or ExternKernels. During scheduling if there is a potential
    epilogue we will benchmark each of the choices with the epilogue to determine an implementation.
    Otherwise, the fastest base choice will be chosen.
    r)  rm  r  rg   choice_timings'Callable[[], dict[ChoiceCaller, float]]unfiltered_choiceslist[ChoiceCaller]r  rF  rr   rs   c                   s>   t  j||d |d || _d | _|| _tdd |D | _d S )N)r)  r  r  r  c                 s  s0    | ]}t |tpt |tjjjo|jV  qd S r   )ru   r  r   	_inductorselect_algorithmZExternKernelCallerZhas_out_variant)r   choicern   rn   ro   r    s    

z/MultiTemplateBuffer.__init__.<locals>.<genexpr>)r  r  _choice_timings_fn_choice_timingsZoriginal_inputsr  _output_plannable)rA  r)  r  r  r  r  r  rn   ro   r    s   zMultiTemplateBuffer.__init__rh   c                 C  rP  )z^
        Are all possible choices TritonTemplates or Extern Kernels with out variants
        )r  rD  rn   rn   ro   output_plannable  s   z$MultiTemplateBuffer.output_plannabledict[ChoiceCaller, float]c                 C  s   | j d u r
|  | _ | j S r   )r  r  rD  rn   rn   ro   r    s   

z"MultiTemplateBuffer.choice_timingscallerr  c                 c  sR    t |tjjjsJ | j|jksJ | j}| | _z	d V  W || _d S || _w r   )ru   r   r  r  TritonTemplateCallerr)  r  r  )rA  r  renderrn   rn   ro   swap_as_triton_caller  s   
z)MultiTemplateBuffer.swap_as_triton_callerc                 C  sJ   t |tjjjs
J |  |jjksJ |  |jj	ksJ |
 | _d S r   )ru   r   r  r  r  r   r)  r   r  r   r  r  )rA  r  rn   rn   ro   finalize_as_triton_caller  s   z-MultiTemplateBuffer.finalize_as_triton_callertuple[ChoiceCaller, float]c                 C  s    t | j| jjd}|| j| fS )Nr  )r^  r  r  )rA  Z
min_choicern   rn   ro   get_min_choice  s   z"MultiTemplateBuffer.get_min_choice)r)  rm  r  rg   r  r  r  r  r  rF  rr   rs   r  )rr   r  )r  r  )r  r  rr   rs   )rr   r  )rj   rk   rl   r  r  r  r  r  r  r  r  r  r  r:  rn   rn   r  ro   r    s    
r  c                      s&   e Zd Zd fddZd	d
 Z  ZS )CUDATemplateBufferworkspace_sizer   templaterW   rr   rs   c                   s    t  ||| || _|| _d S r   )r  r  r  r  )rA  r)  r  r  r  r  r  rn   ro   r    s   
zCUDATemplateBuffer.__init__c                 C  s   | j d ur| j S dS r  )r  rD  rn   rn   ro   r    r   z%CUDATemplateBuffer.get_workspace_size)r  r   r  rW   rr   rs   )rj   rk   rl   r  r  r:  rn   rn   r  ro   r    s    r  c                      s,   e Zd Zd fddZd	 fddZ  ZS )
CppTemplateBufferrr   rs   c                   s&   t  ||| || _|| _d | _d S r   )r  r  r  r  r  )rA  r)  r  r  r  r  r  rn   ro   r    s   
zCppTemplateBuffer.__init__rm  c                   sV   t | jtr&t | jtsJ | jd }t |tsJ |j}t |ts$J |S t  S r  )	ru   r)  MultiOutputLayoutr  r   ru  rm  r  r   )rA  Zfirst_outputr)  r  rn   ro   r     s   

zCppTemplateBuffer.get_layoutr  r  )rj   rk   rl   r  r   r:  rn   rn   r  ro   r    s    r  c                   @  sX   e Zd ZU ded< dddZddd	ZedddZedd Z	dddZ
dddZdS )r  r  r  rr   r  c                   s   t tj  }tj | jD ]#}t|tr | fdd|D  qt|tr&q|	 |
  qt tj  fdd|  D }tj||t  dS )Nc                 3      | ]	} |  V  qd S r   rS  r  ri  rn   ro   r    r  z/InputsKernel.get_read_writes.<locals>.<genexpr>c                 3  r  r   rS  r  r  rn   ro   r        
)r  writesZindex_exprs)r+   r2   r6   ri  r  ru   rv   updater   r\  r  r  Z
ReadWrites)rA  r  inputr  rn   r  ro   r    s    


zInputsKernel.get_read_writesr  c                 C  r  r   r  rD  rn   rn   ro   rM  %  r  zInputsKernel.get_readsr   rb   c                 C  sz   t |tr|j}t |tr|j}t |trt |tst|}t |tr)| |S t |t	r0|S t |t
tfs;J ||S r   )ru   ra   r(  rt  ry  r-  r  realize_inputunwrap_storage_for_inputTorchBindObjectru  r#  r   rn   rn   ro   r  (  s   





z%InputsKernel.unwrap_storage_for_inputc                 C  s@   g }| D ]}t |trdd |D }nt|}|| q|S )Nc                 S  r  rn   )r  r  r   rn   rn   ro   r   @  r   z/InputsKernel.unwrap_storage.<locals>.<listcomp>)ru   rv   r  r  r  )r  Z
inputs_newr   rn   rn   ro   r  ;  s   

zInputsKernel.unwrap_storagerh   c                 C  r  r  rn   rD  rn   rn   ro   r  F  r   zInputsKernel.is_externr   c                 C  r  r  rn   rD  rn   rn   ro   r  I  r   zInputsKernel.num_readsNr  r  )r   rb   rr   rb   r  r  )rj   rk   rl   rm   r  rM  r8  r  r  r  r  r  rn   rn   rn   ro   r    s   
 




r  c                   @  s    e Zd Zd	ddZd
ddZdS )	NopKernelrr   rh   c                 C  r  r  rn   rD  rn   rn   ro   r  N  r   zNopKernel.is_no_opr  c                 C  r  r   r*   rD  rn   rn   ro   rM  Q  rR  zNopKernel.get_readsNr  r  )rj   rk   rl   r  rM  rn   rn   rn   ro   r  M  s    
r  c                   @  s@   e Zd ZdZedd ZedddZedd ZdddZdS )ConcatKernelzn
    There isn't actually a real kernel for concat, we just change the
    storage for the upstream data.
    c                 C  s  |d   }|d  }t|d  }dg}|| g}d|  kr)t|k s,J  J tdt|D ]Z}||  }	|||  t|	t|ksLJ ||  |ksVJ ||   |ks`J tt|D ]}
|
|krw||
 |	|
  ||
< qftjj	
||
 |	|
 ||
< qf|||  q3t|}tjrt|||d j}tt|D ]!}|| }t|r| }t|trt|j|jrt|} nqtdd |D }tjjjd }t|tsJ |du rtdd |D rt|}td t||||dg d}t|}g }tt|D ]N}| || t j!|||| || dd	}|j"| t|| j#t$r6|| j#% }n|| j#}|& rVt'||   j(rVt)|sV||*  q	t|dkrntj+|t,j-rntj.| tj/||_0| 1|j"|_"tj2| |S )
Nr   r0   c                 s  r
  r   )r   r  rn   rn   ro   r    r  z&ConcatKernel.create.<locals>.<genexpr>Fc                 s  sB    | ]}d |j v o|j d  jtjdp|j d  jtjdV  qdS )r  r  N)r
  rk  r   r  r  r   argrn   rn   ro   r    s    

)r   r   r   r   r   r)  r  )r  )3r   r   rv   r   r   r   r  rU   r   r   r  r   r   r1   r  rm  r  r   r   r   ru   r+  r  r   r   r!   r]  r  r   r  rt  rI  r  r"  r  r(  ry  r  is_input_bufferrM   r   rL   r  r  r3   ZFOREACHZregister_operation_listr  r   r  r  )r#  r  r  r   r   r  Zoffsets_startZoffsets_endr   Z
input_sizer  Zoutput_strider   r)  Zany_input_is_storage_and_layoutZfx_node_argsZconcat_kernelkernelZop_namesZinput_bufferZinput_unwrappedrn   rn   ro   r"  [  s   
 



 zConcatKernel.createNc                 C  s   t |tr| |j|S t |jtrCt |jjtr|jjsdS |d u r%dS t|	 t|	 ks3dS t
dd t|	 |	 D S t |jjtoPt |jt S )NFTc                 s  r#  r   r$  r%  rn   rn   ro   r    r&  z=ConcatKernel.can_realize_into_without_copy.<locals>.<genexpr>)ru   ra   can_realize_into_without_copyr(  r  r)  r+  r  r   r  r  r   r   ExternKernelAlloc)r#  rG  rH  rn   rn   ro   r    s$   
z*ConcatKernel.can_realize_into_without_copyc              	   C  s   t |tst|rt|\}}t||d}t |tsJ |t |tr*| |j|S t |trJ|  t	|jds;J | 
||rJt||j_|jS tj| | | dd t| | D d}| ||S )Nr'  r)  c                 S  rC  rn   rD  rE  rn   rn   ro   r     rF  z-ConcatKernel.realize_into.<locals>.<listcomp>rP  )ru   r-  r   r*  ra   rI  r(  rt  r  r  r  r)  r)  rA  r"  r   r   r  r   r   )r#  rG  rH  r.  r)  pwrn   rn   ro   rI    s,   


	zConcatKernel.realize_intorr   rh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   zConcatKernel.should_allocater   r  )	rj   rk   rl   r  r8  r"  r  rI  rV  rn   rn   rn   ro   r  U  s    
`
 r  c                      s&  e Zd ZU dZded< ejedZded< dZ	ded	< dZ
d
ed< dZd
ed< ejedZded< dZded< dZded< dZded< ejedZded< ejedZded< 							dfdg fddZdhddZdid!d"Zd#d$ Zd%d& Zdgd'd(Zd)d* Zdjdkd+d,Zdld-d.Zd/d0 Zed1d2 Zedmd4d5Z ed6d7 Z!ed8d9 Z"ed:d; Z#e			<dndodAdBZ$edpdCdDZ%edpdEdFZ&edGdH Z'edIdJ Z(edKdL Z)dgdMdNZ*dOdP Z+djdqdSdTZ,dUdV Z-dWdX Z.dpdYdZZ/dgd[d\Z0d]d^ Z1d_d` Z2didadbZ3drdddeZ4e4Z5  Z6S )sr  rn   ztuple[Any, ...]constant_argsr9  zdict[str, Any]r   NzOptional[ReinterpretView]output_viewr   python_kernel_namecpp_kernel_namezIterable[str]ordered_kwargs_for_cpp_kernelzFOptional[Union[torch._ops.OpOverload, torch._ops.HigherOrderOperator]]op_overloadzOptional[list[dict[str, Any]]]arg_propertiesz#Optional[dict[str, dict[str, Any]]]kwarg_propertiesz"dict[sympy.Symbol, pytree.KeyPath]unbacked_bindingszlist[MutationOutput]mutation_outputsrr   rs   c                   sn   t  j|||d || _|r|ni | _|| _|
| _| | | | |	| _| 	  i | _
g | _tjj| _d S Nr  )r  r  r  r   r  r  set_cpp_kernel_nameset_python_kernel_namer  collect_arg_kwarg_propertiesr  r  rU   r   r  Zfx_node)rA  r   r)  r  r  r   r  r  r  r  r  r  rn   ro   r    s    

zExternKernel.__init__r  c                 C  s   | g| j S r   )r  rD  rn   rn   ro   r  4  r   zExternKernel.get_outputsr   c                 C  r  r   r*   rD  rn   rn   ro   r  7  rR  z%ExternKernel.get_unbacked_symbol_defsc                 C  s   t | jtjjrdd | jjjD ndd tt| j	D | _
t | jtjjr1dd | jjjD ni | _t | jtjjrW| jsJdd | jjjD | _dd | jjjD | _d S d S )Nc                 S  s$   g | ]}|j s|j|j|jd qS ))r   r   r  )
kwarg_onlyr   	real_typer  r  rn   rn   ro   r   >  s    z=ExternKernel.collect_arg_kwarg_properties.<locals>.<listcomp>c                 S  s   g | ]}i qS rn   rn   r   rn   rn   ro   r   H  r   c                 S  s   i | ]}|j |j|jd qS ))r   r  )r   r  r  r  rn   rn   ro   r   K  r  z=ExternKernel.collect_arg_kwarg_properties.<locals>.<dictcomp>c                 S     g | ]}|j r|jqS rn   r  r   r  rn   rn   ro   r   V  
    c                 S  s   g | ]}|j r|qS rn   )r  r  rn   rn   ro   r   Y  s
    )ru   r  r   _ops
OpOverload_schema	argumentsr   r   r  r  allarg_propertiesr  schema_kwargsrD  rn   rn   ro   r  :  s*   

z)ExternKernel.collect_arg_kwarg_propertiesc                 C  s$   t | jtr|   |   d S d S r   )ru   r)  r   apply_constraintr  rD  rn   rn   ro   r  ]  s   zExternKernel.decide_layoutc                 C  s$   t | |\}}|r|| d S d S r   )rJ   r  )rA  wrapperZ
origin_strZ_detailed_origin_strrn   rn   ro   codegen_commentb  s   zExternKernel.codegen_commentc                 C  r  r   r  rA  r  rn   rn   ro   codegeng  r   zExternKernel.codegenc                 C  s   || _ tjjrt| jtjjsd S | j}| j d u rB|j	dkr;|j
dkr+|jdd n|jdd}d| d| _ d S |jj| _ d S d S )Natenr  .r   r   z
at::_ops::z::call)r  rU   r   cpp_wrapperru   r  r   r  r  	namespaceZ_overloadnamerj   r  replacer   r   )rA  r  r  opnamern   rn   ro   r  j  s   




z ExternKernel.set_cpp_kernel_namec                 C  sd   || _ |d ur	d S | j}|d u rd S t|tjjr"d|j | _ d S |jdd d|j | _ d S )Nztorch.ops.higher_order.z._ops.z.ops.r
  )	r  r  ru   r   r  HigherOrderOperatorrj   rk   r  )rA  r  r  rn   rn   ro   r    s   z#ExternKernel.set_python_kernel_namec                 C  s:   |    }r	|jntjj}tjjrtjj| j|S | j	S r   )
r   r   rU   r   device_typer  r  Zget_c_shim_func_namer  r  )rA  dr   rn   rn   ro   get_kernel_name  s   zExternKernel.get_kernel_namec                 C  s:   t j|  |  |  |  |  |  d}|  |S )N)r   r   r  r	  r:  r8  )	rA  r"  r   r   r  r   rT  rQ  r  )r   r  rn   rn   ro   
copy_input  s   zExternKernel.copy_inputituple[Any, list[Any], list[Any], Callable[[Any, Any], Any], Optional[dict[sympy.Symbol, pytree.KeyPath]]]c                   sx  ||d}t |\} g g }g }|D ]/}t|to"t|t  d r.|| qt|tjr>tj	j
jj|d d}|| q fdd}	fdd|D }|D ]}
t|
rbt|
dd	 qVg }|D ]n}
t|
ts|
 tj	jv r|tj	j|
   qgt|
ts|
 tj	jv r|tj	j|
   qgt|
tr||
  qgt|
tjjjr|
jj}|
jjd
kr|d usJ |tjj|   qg|t|
dd qg|	||\}}||i |}d }tjj }rt |tj!| t"||tj!j#$d}t|t%t&fs	|gn|}|D ]'}t|tj'r3|j(r3d}tj	j!j#$dd  }r/| d| }|tj	_)q||||	|fS )Nr   r  )r  c                   sd   g }t | }t |}D ]}|r|t| q|t| qt| }|dg |di fS )Nr   r   )iterr  nextpytreeZtree_unflattenr  )Znew_tensor_argsZnew_non_tensor_argsr>  Z
it_tensorsZit_non_tensorsZ	is_tensorr%  )	args_specis_arg_tensorrn   ro   unflatten_args  s   z3ExternKernel.process_kernel.<locals>.unflatten_argsc                   r  rn   r  r  r#  rn   ro   r     r   z/ExternKernel.process_kernel.<locals>.<listcomp>Trf  r  )r   r  zEsparsity not handled. Please file issue for sparse inference weights.stack_tracez Found from : 
 )*r  tree_flattenr  ru   rb   GeneratorStater}   r   rU   r   r   r   Zcreate_symintnoder   r*  ry  r  	constantsZtorchbind_constantsr  get_real_objr   r  Zirr   r   r   r  Zdefault_generatorsZclone_stater   	fake_moder&   r  r$   r
  r  rv   rw   TensorZ	is_sparseZdisable_cudagraphs_reason)r#  r  r   r   Zbinded_argsZ	args_flattensor_argsnon_tensor_argsr  r  r   Zexample_argsZdevice_indexnew_argsZ
new_kwargsexample_outputr  r   Zexample_out_lir   msgr  rn   )r  r#  r  ro   process_kernel  s~   



zExternKernel.process_kernelc              	   C  sV  t |tsJ t |tr|S | }tj| }|dus J | }|durQd|j	v rQt |j
trQ|j	d jtjdsG|j	d jtjdrQ|t|  n|  tj| dd\}}|d }| |}tjj||}tjj||}	tjj||}
t||	|
 }||krtd|	|
| tt|jt |! |" | |	|
dd	S )
z
        In order to pass this to an extern kernel we need a
        ReinterpretView not a View.  This allows us to avoid some
        unneeded copies.
        Nr  r  r%  r_   r   z@convert_to_reinterpret_view failed: stride=%s offset=%s index=%sr  r'  )#ru   ry  r-  r  rU   r   r=  r  rT  r
  r)  r   rk  r   r  r  r  r!   r   r  r2   r  r  r   r  Zstride_varsZ
offset_varrN   r  r  rj  r(  r+  r  r   )r#  r   Zx_unwrap_viewr  Zx_unwrap_view_fx_nodeZ
index_argsrv  r  r   r!  r,  expectedrn   rn   ro   convert_to_reinterpret_view  sf   






z(ExternKernel.convert_to_reinterpret_viewc                 C  s  |d u rt  S t|tjtjjjtfrt|dS t|t	r.t
jtj|j| | dS t|tr5|S t|tr@| |jS t|trQt| |j| dS t|trp|  t| rpz| |W S  tyo   Y nw t|tr{|  |S t|ttfr|S |  |S )N)r  )r   r   r'  )!r_  ru   r}   r   r~   r   r   r   r   r  rU   r   Zadd_tensor_constantr   r  r   r   r   rQ  ra   r  r(  r-  r   ry  r  r   r  r+  rj  rt  NonTensorObjr  r  rn   rn   ro   r  Y  s<   







zExternKernel.realize_inputc                 C  sD   t |rt| dkr|S | D ]
}|dkr|  S q| |S r  )r   r   r  r  )r#  r   r   rn   rn   ro   require_stride1z  s   
zExternKernel.require_stride1Fr   Optional[Sequence[int]]r  rz  c              	     s  |d us
 d us
J   dv r sS trt trI|r=tddt|r6ttj	j
 jn||d S tddd | d S t ttfrs|r[ |sh rst  j rs d urqt S S t trt  trtdt  tr|r  |s rt   j rS ttr|r |sǈ rt  j rɈS ttrtjtrtjtst rt jtsz!| j_|r| j||dW S  r| j |dW S W n
 t y   Y nw d } } d urItj	j
 fdd	t!t" D }|D ]}t#j$j%&|d
dq;| 'tdd|| d |ret|scJ S |r|d urr d ustJ t#j$j%(|t S S )N)r   r0   TF)rg  rn  ro  r  rr  zHthe MutationLayoutSHOULDREMOVE's real layout shouldn't be FlexibleLayoutrs  c                   s4   g | ]}  | d r | dr|qS )r   r   )r  r  r   r   r  r   r   rn   ro   r     s    z0ExternKernel.require_strides.<locals>.<listcomp>r   r0   ))r  r   ru   r   r   r*  rx  r   rU   r   r   Z
size_hintsr   r+  r)  rw  r  r   r2  r<  r?  r  rZ  ra   r(  ry  r-  r  r  r+  require_stride_orderr  rj  r   r   r   r  ZloweringZslice_r  r  )r#  r   r   r  r  Zexpanded_dimsZ	orig_sizer  rn   r/  ro   require_strides  s   
	


	


zExternKernel.require_stridesc                 C     | j |||dS )N)r  r  r1  )r#  r   r  r  rn   rn   ro   r  !  s   z"ExternKernel.require_exact_stridesc                 C  r2  )N)r   r  r3  )r#  r   r   r  rn   rn   ro   r0  '     z!ExternKernel.require_stride_orderc                 C     |  |tS r   )r0  r  r  rn   rn   ro   require_channels_last+  r  z"ExternKernel.require_channels_lastc                 C  r5  r   )r0  r  r  rn   rn   ro   require_channels_last_3d/  r  z%ExternKernel.require_channels_last_3dc              	   C  s    |  |tttt| S r   )r0  rv   r  r   r   r   r  rn   rn   ro   require_contiguous3  s    zExternKernel.require_contiguousc                 C  r   r   rn   rD  rn   rn   ro   r  7  r   zExternKernel.apply_constraintc                 C  s   t |ttfs	J t |trt|}| jsJ dt|}t| j}||k rQtd| j||  t||D ]}| j| d }|	||v rH|| n| j| d  q5|S )Nz/ExternKernel.arg_properties should not be emptyzv%s has %d unprovided positional arguments. Will check if they are in the keyword arguments or will use default values.r   r  )
ru   rv   rw   r  r   r  r  r  r   r  )rA  r   r   Zn_argsZ
n_pos_argsr   arg_namern   rn   ro   fill_non_provided_args:  s(   	

z#ExternKernel.fill_non_provided_argsr  r7  c           	      C  s   t jjrig }d }|r"| jr"t| jt|ksJ ddd | jD }t| jD ]?\}}|d ur@||| }|r=|dnd }nt| j| }| jrY|t| jk rY| j| dnd }|	t jj
|| q'|S tt jj
j| jS )NzDnames passed to codegen_const_args does not match self.constant_argsc                 S  s   i | ]}| d |qS r   )r  r  rn   rn   ro   r   h  r  z3ExternKernel.codegen_const_args.<locals>.<dictcomp>r   )rU   r   r  r  r   r  r   r  r  r  r  val_to_arg_strra  )	rA  r  r>  Zname_to_arg_propertiesr   r   proptype_r   rn   rn   ro   codegen_const_args\  s0   
zExternKernel.codegen_const_argsc                 C  s   t jjr| jd ur| g | j| j| j}d}n| j}d}g }t|D ]4\}}t jjrN| j	r6|t
| j	k s:J d| j	| d}|t jj|| q$|t jj| q$|rb||   |S )NFTz-Invalid access to ExternKernel.arg_propertiesr   )rU   r   r  r  r:  r  r  r   r   r  r   r  r  r  r;  r  r>  )rA  r  Zneed_codegen_constant_argsr   r   r   r=  rn   rn   ro   codegen_args|  s&   zExternKernel.codegen_argsc                 K  sX   ||v r	| |S || jv r| j |S | jr%|| jv r%| j | dS t| d)zGiven an argument name, queries for values in (in order):
        1. any provided kwargs for this function.
        2. the class self.kwargs member.
        3. any available default arguments in self.allarg_properties.r  z not in self.allarg_properties)r  r   r  r  )rA  r9  r   rn   rn   ro   get_kwargs_value  s   

zExternKernel.get_kwargs_valuec                 C  s   t jjrR| jd urt| jdkrg S g }| jD ]8}|r |dkr q| |}t|t	j
r1|| q| jrB|| jv rB| j|dnd }|t jj|| q|S dd | j D }|S )Nr   r   r   c                 S  s(   g | ]\}}| d t jj| qS r  rU   r   r  r;  )r   krk  rn   rn   ro   r     s    z/ExternKernel.codegen_kwargs.<locals>.<listcomp>)rU   r   r  r  r   r  r  r@  ru   r}   r   r  r  r  r  r;  r   ru  )rA  skip_outr   r9  rk  r=  rn   rn   ro   codegen_kwargs  s,   


zExternKernel.codegen_kwargsc              	   C  st   t jr6tjjs8t|  dkrd S tjj|  }tjj| 	 }|
d|   d| d| d d S d S d S )Nr   zassert_size_stride(r  r   )r1   Zsize_assertsrU   r   r  rQ   r   r  Zcodegen_shape_tupler  r  r  )rA  r  r   r   rn   rn   ro   codegen_size_asserts  s   z!ExternKernel.codegen_size_assertsc                 C  s   |   }|  }|g g|fS )zD
        get output sizes and strides, for template_codegen
        )r   r  )rA  _sizeZ_stridern   rn   ro   get_group_stride  s   zExternKernel.get_group_stridec                   s  t jj|  }|  }fdd|D }dd tt|D ttt||jdd}dd t	|D fddttD }fd	d|D | 
 }|}t jj||g\}}}	td
\}
 tt| fdd|D }tt||}|t|fS )zC
        Manually get canonicalization of the output index
        c                   r  rn   )r   r  )r   rn   ro   r     r   z-ExternKernel.canonicalize.<locals>.<listcomp>c                 S  s   g | ]	}t d | qS )r  )rO   r   rn   rn   ro   r     r  T)r  r  c                 S  r   rn   rn   r   rn   rn   ro   r     r   z-ExternKernel.canonicalize.<locals>.<dictcomp>c                   r   rn   rn   r   r   rn   ro   r     r   c                   r   rn   rn   r   )rr  rn   ro   r     r   cc                   r   rn   rn   r  )add_varrn   ro   r     r   )rU   r   r   r   r  r   r   r  r!  r   r  r  r:   rx   r   rR   r}   r  rw   )rA  r  r!  Zindex_orderr   rD  r   Z	new_sizesr   r  r   replacementrn   )rI  rr  r   r   ro   canonicalize  s$   
 zExternKernel.canonicalizec                 C  sD   t tj  }| jD ]}|t|O }q	| j D ]}|t|O }q|S r   )r+   r}   r   r  maybe_free_unbacked_symbolsr   ry   )rA  r%  r  rn   rn   ro   r    s   
z%ExternKernel.get_unbacked_symbol_usesr   c                   sP   t  dd }d|g}| fddt D 7 }|d j  |S )Nr  zpython_kernel_name=c                   s$   g | ]}|j  d t |j  qS r  )r   r   )r   r  rD  rn   ro   r     s    z(ExternKernel.__str__.<locals>.<listcomp>r  )r   r  fieldsr  r:  re  )rA  kernel_namer]  rn   rD  ro   r    s   
zExternKernel.__str__rn   NNNNrn   Nr  r  r  r   r  r   rr   rs   )r  r   rr   rs   )rr   r  )NNF)r   r.  r  rz  r  )r  r7  r  )7rj   rk   rl   r  rm   r  r  rx   r   r  r  r  rv   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r8  r)  r+  r  r-  r1  r  r0  r6  r7  r8  r  r:  r>  r?  r@  rD  rE  rG  rK  r  r  r7  r:  rn   rn   r  ro   r     s   
 


#


m
D
 
	 



" 

	


r  c                      sB   e Zd ZdddZ							dd fddZdd
dZ  ZS )ExternKernelOutrr   rs   c                 C  s   |  | g |  | jdd}|  }tjjr!| jdkr!d}n|  }|   }r.|j	ntjj
}|||  | jr@| j nd || d S )NT)rC  ztorch::inductor::_mm_plus_mmZaoti_torch__mm_plus_mm_out)r  r?  rD  r  rU   r   r  r  r   r   r  Zgenerate_extern_kernel_outr  r  )rA  r  r   rN  r  r   rn   rn   ro   r    s    

zExternKernelOut.codegenrn   Nc
           
        sF   t  d || |||pi d ||||	
 tj| | _tj|  d S r   )r  r  r  rU   r   r  r   r  )
rA  r)  r  r  r   r  r  r  r  r  r  rn   ro   r  #  s   zExternKernelOut.__init__rh   c                 C  r  r  rn   rD  rn   rn   ro   rV  >  r   zExternKernelOut.should_allocater  rO  r  )rj   rk   rl   r  r  rV  r:  rn   rn   r  ro   rQ    s    
rQ  c                      s   e Zd Zd	 fddZ  ZS )
RandomSeedscountr   r   r  rr   rs   c                   sF   t t j}t jt|t j|gdg |j|j|ggddtj	j
d d S )Nr  zaten.randint.low_outzat::_ops::randint_low_out::call)r)  r  r  r  r  r  )r   r  r  r  r  r+  r^  r2  r	  randintZlow_out)rA  rS  r   Zlimitsr  rn   ro   r  C  s   
zRandomSeeds.__init__)rS  r   r   r  rr   rs   rj   rk   rl   r  r:  rn   rn   r  ro   rR  B      rR  c                      sH   e Zd ZdddZ						dd fddZdd
dZdd Z  ZS )r  rr   rs   c                 C  sL   |  | g |  |  }tjj| | t| jt	r$| 
| d S d S r   )r  r?  rD  rU   r   r  Zgenerate_extern_kernel_allocru   r)  rm  rE  rA  r  r   rn   rn   ro   r  W  s   
zExternKernelAlloc.codegenrn   Nc	           	        sL   t  d || |||pi d ||||
 g | _tj| | _tj|  d S r   )	r  r  r  r  rU   r   r  r   r  )	rA  r)  r  r  r   r  r  r  r  r  rn   ro   r  ^  s   zExternKernelAlloc.__init__rh   c                 C  r  r  rn   rD  rn   rn   ro   rV  |  r   z!ExternKernelAlloc.should_allocatec                 C  r  r   r  rD  rn   rn   ro   r    r   z"ExternKernelAlloc.apply_constraintr  )rn   NNNrn   Nr  )rj   rk   rl   r  r  rV  r  r:  rn   rn   r  ro   r  V  s    

r  c                      s>   e Zd ZdZd fddZddd	Zd
d ZdddZ  ZS )r  zP
    An output buffer that represents the mutation of a pre-existing buffer
    mutating_noder  rr   rs   c                   sD   t  jd |d | }tj| |g| _|| _tj| | _	d S r^  )
r  r  r  rU   r   r>  mutation_namesrX  r  r   )rA  r)  Zmutated_noderX  Zmutated_node_namer  rn   ro   r    s   zMutationOutput.__init__c                 C  rP  r   )rX  rD  rn   rn   ro   rV    rR  zMutationOutput.get_defining_opc                 C  rP  r   )rY  rD  rn   rn   ro   r    rR  z!MutationOutput.get_mutation_namesrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   zMutationOutput.should_allocate)rX  r  rr   rs   rX  r  )	rj   rk   rl   r  r  rV  r  rV  r:  rn   rn   r  ro   r    s    
r  c                      sP   e Zd ZU dZi Zded< e	ddddZ	dd fddZdddZ	  Z
S )TMADescriptora$  
    An IR node representing a host-side TMA descriptor in the Triton API
    (the ones obtained via create_{1d,2d}_tma_descriptor calls). Mostly
    useful for user-defined Triton kernels relying on host-side TMA; but
    can, in principle, be used for Inductor's Triton templates, too.
    zdict[Any, TMADescriptor]_CACHENr  rb   r  list[Union[int, torch.SymInt]]
block_dimselement_sizer  c                 C  s8   t ||||f}|| jvrt||||| j|< | j| S r   )idr[  rZ  )r#  r  r  r]  r^  r  rn   rn   ro   r"    s   

zTMADescriptor.createrr   rs   c                   s   t |dv sJ t |t |ksJ |d u r| j}|| _|| _|| _|| _t | j| _|g}g | j| j| j}t 	d t
t|| d|t|d  tj| | _tj|  d S )N)r0   r   r'  )r   r   r  r  r  r]  r^  r  r  r  r)  r-  r   rw   rU   r   r  r   r  )rA  r  r  r]  r^  r  r  r  rn   ro   r    s>   
zTMADescriptor.__init__c                 C     | |  d S r   )Zgenerate_tma_descriptorr  rn   rn   ro   r    r  zTMADescriptor.codegenr   )r  rb   r  r\  r]  r\  r^  r  )
r  rb   r  r\  r]  r\  r^  r  rr   rs   r  )rj   rk   rl   r  r[  rm   r8  r"  r  r  r:  rn   rn   r  ro   rZ    s   
 	-rZ  c                      s\   e Zd Zdd ZdddZd fdd	Zdd
dZd fddZdddZdddZ	  Z
S )UserDefinedTritonKernelc                   s   ddl m} ddlm} || j g }g }g }t |ret dr0| fdd j	D  nt ds7J | j
 t drR jD ]}| jj|  qEnt d	sYJ | j  j} j  |||fS )
Nr   )	Autotuner)kernel_side_tablerestore_idxc                 3  s    | ]	} j j| V  qd S r   )r   	arg_namesr   r  rn   ro   r    r  zBUserDefinedTritonKernel.get_kernel_and_metadata.<locals>.<genexpr>restore_value	reset_idxreset_to_zero)Ztriton.runtime.autotunerrb  *torch._higher_order_ops.triton_kernel_wraprc  Z
get_kernel
kernel_idxru   r  r  rd  rg  rh  r  r   re  ri  configs)rA  rb  rc  rl  restore_value_argsreset_to_zero_argsr   rn   rf  ro   get_kernel_and_metadata  s,   




z/UserDefinedTritonKernel.get_kernel_and_metadatarr   rs   c              	     s  ddl m}  \ }}}| |j||j\}}}fddjD }	t fdd jD }
g }g }g }t	
|	 tt	d|D ]f\}}|| t|trd||  ||  qGt|ttttjfr{|| |t| qG||
v r|d |t qG|d u r	 | r|d |t qG|  qGtd	t| d
| | |j|||||d d d S )Nr   )triton_version_uses_attrs_dictc                   s   i | ]}|  |qS rn   r@  )r   rB  rD  rn   ro   r     r  z3UserDefinedTritonKernel.codegen.<locals>.<dictcomp>c                   r  rn   )re  r   rf  rn   ro   r     r   z3UserDefinedTritonKernel.codegen.<locals>.<listcomp>rY  r  zUnsupported arg type: r   T)	arg_typesZraw_argstriton_metar  r   )Ztorch._inductor.utilsrp  ro  Z!define_user_defined_triton_kernelr   gridr  r+   Z
constexprsr  r  ru  r   repeatr  ru   rb   r  r   r   r?  rh   r}   r   r   r!  rj  r  Zgenerate_kernel_callr   )rA  r  rp  rl  rm  rn  new_namers  Zextra_launch_argsZ
named_argsZconstexpr_namesr   rr  Zraw_args_filteredr   r  rn   )r  rA  ro   r    sp   
	



	



zUserDefinedTritonKernel.codegenr   c                   s   t   t| jB S r   )r  r  r%   rt  rD  r  rn   ro   r  O  s   z0UserDefinedTritonKernel.get_unbacked_symbol_usesc                 C  r  r   r*   rD  rn   rn   ro   r  T  rR  z0UserDefinedTritonKernel.get_unbacked_symbol_defsc                  sZ  g }i }g }   D ]3\}}	t|	tr4t|	}
||v r*tj|
g|| R  }
||
 |
||< q
||	 |	||< q
t	|dksFJ |d 
 _t d tjd|t|| |_|_ \}}}} fdd|jD _ddlm} t	|dkr|d jni } fdd||i  |D _fddjD _tj d S )Nr   r  c                   s   g | ]}| v r|qS rn   rn   r  kernel_argsrn   ro   r   x  r  z4UserDefinedTritonKernel.__init__.<locals>.<listcomp>)identify_mutated_tensorsc                   r   rn   rn   r   r  rw  rn   ro   r     s    c                   s    g | ]}t t jd | qS r  )r  r7  r   r  rD  rn   ro   r     s    )ru  ru   ra   r  r  r  rZ  r"  r  r   r   r   r  r  r7  rw   rk  rt  ro  re  r  rj  ry  r   Zmutable_argsr  rU   r   r  )rA  rk  rt  Ztma_descriptor_metadatarx  r  r   r  rB  rk  r   r  rl  r   ry  Zautotuned_kwargsr  )rx  rA  ro   r  W  sL   








z UserDefinedTritonKernel.__init__r  c                 C  rb  r   )rv   r  rD  rn   rn   ro   r    r  z#UserDefinedTritonKernel.get_outputsr  c                 C  rP  r   r  rD  rn   rn   ro   r     rR  z"UserDefinedTritonKernel.get_devicer  r  r  r  )rj   rk   rl   ro  r  r  r  r  r  r   r:  rn   rn   r  ro   ra    s    
K

5ra  c                      H   e Zd ZdZdddZdddZd	d
 ZdddZd fddZ  Z	S )InplaceBernoulliFallbackE
    This needs to be a custom class to handle mutation properly
    rr   rs   c                 C  s   dd | j D \}tjjr)||   d| ddtt| j	 d|j
  d S ||   d| ddtt| j	 d|j
  d S )Nc                 s      | ]}|  V  qd S r   r  r   r   rn   rn   ro   r    r  z3InplaceBernoulliFallback.codegen.<locals>.<genexpr>r`  r  z, NULL)r   )r  rU   r   r  r  r  rc  ra  reprr  ending)rA  r  r   rn   rn   ro   r    s   ,,z InplaceBernoulliFallback.codegenrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   z(InplaceBernoulliFallback.should_allocatec                 C     | j d  gS r  r  r  rD  rn   rn   ro   r    r  z+InplaceBernoulliFallback.get_mutation_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r    rR  z1InplaceBernoulliFallback.get_unbacked_symbol_defsc                   sV   t  jd t| d| |g||d tj|  tj	| | _
tj|  d S )Nr  r  )r  r  r7  r   r  rU   r   r>  r  r  r   r  )rA  r  r   r  r  rn   ro   r    s   
z!InplaceBernoulliFallback.__init__r  r  r  
rj   rk   rl   r  r  rV  r  r  r  r:  rn   rn   r  ro   r|        


r|  c                      sX   e Zd ZdZdddZdddZd	d
 ZdddZd fddZe	ddddZ
  ZS )InplaceCopyFallbackr}  rr   rs   c                 C  s    |   \}}}|||| d S r   )r?  codegen_device_copy)rA  r  rH  rG  non_blockingrn   rn   ro   r    s   zInplaceCopyFallback.codegenrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   z#InplaceCopyFallback.should_allocatec                 C  r  r  r  rD  rn   rn   ro   r    r  z&InplaceCopyFallback.get_mutation_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r    rR  z,InplaceCopyFallback.get_unbacked_symbol_defsc                   sJ   t  jd |||ddd tj|d   tj| | _tj|  d S )Nz
aten.copy_Zaoti_torch_copy_)r  r  r   )	r  r  rU   r   r>  r  r  r   r  )rA  r)  r  r  r  rn   ro   r    s   zInplaceCopyFallback.__init__Fr  c                   s6    fdd||fD }|f}t t| d||}|S )Nc                   r  rn   r  r  r  rn   ro   r     r   z.InplaceCopyFallback.create.<locals>.<listcomp>r  )r  r7  r   )r#  rH  rG  r  r  r  r>  rn   r  ro   r"    s   zInplaceCopyFallback.creater  r  r  r  )r  rh   )rj   rk   rl   r  r  rV  r  r  r  r8  r"  r:  rn   rn   r  ro   r    s    


r  c                   @  s@   e Zd ZdZdddZdddZd	d
 ZdddZdddZdS )MutatingFirstArgExternKernelr}  rr   rs   c                 C  sJ   g dd | j D tt| j}||   dd| d|j  d S )Nc                 s  r~  r   r  r  rn   rn   ro   r    r  z7MutatingFirstArgExternKernel.codegen.<locals>.<genexpr>r`  r  r   )r  ra  r  r  r  r  rc  r  )rA  r  Zargrefsrn   rn   ro   r    s   
z$MutatingFirstArgExternKernel.codegenrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   z,MutatingFirstArgExternKernel.should_allocatec                 C  r  r  r  rD  rn   rn   ro   r    r  z/MutatingFirstArgExternKernel.get_mutation_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r    rR  z5MutatingFirstArgExternKernel.get_unbacked_symbol_defsc                 C  r  r  rn   rD  rn   rn   ro   has_side_effects  r   z-MutatingFirstArgExternKernel.has_side_effectsNr  r  r  )	rj   rk   rl   r  r  rV  r  r  r  rn   rn   rn   ro   r    s    

	
r  c                      s   e Zd Zd fddZ  ZS )ResizeStorageBytesrr   rs   c                   s   t |ts	J dt jd t| d| |g|fd tj	|
  tj| | _tj|  d| _d| _tjj|j
  d S )NzTODO: dynamic shapesr  )r  z"inductor_ops.resize_storage_bytes_z&torch::inductor::resize_storage_bytes_)ru   r   r  r  r7  r   r  rU   r   r>  r  r  r   r  r  r  never_reuse_buffersr\  r(  )rA  variabler  r  rn   ro   r  	  s   
zResizeStorageBytes.__init__r  rU  rn   rn   r  ro   r    rV  r  c                      s(   e Zd Zd fddZd	ddZ  ZS )
SetSourceTensorKernelrr   rs   c                   s   |   t j| ||gdtjjjjd t	j
j|j  t	j
j|  t	j
j|   | }tt|d|| tt|d|| g| _d S )Nz!torch.ops.aten.set_.source_Tensor)r  r  r  )r  r  r  r   r   rS   r	  set_Zsource_TensorrU   r   r  r\  r(  r  r   r  r7  r  )rA  Zself_tensorZstorage_tensorr   r  rn   ro   r    s   

zSetSourceTensorKernel.__init__rX  c                 C  s   | j d  | j d  gS r  r  rD  rn   rn   ro   r  +  s   z2SetSourceTensorKernel.get_inputs_that_alias_outputr  r  )rj   rk   rl   r  r  r:  rn   rn   r  ro   r    s    r  c                      sP   e Zd ZdZdddZdddZd	d
 ZdddZdddd fddZ  Z	S )ScatterFallbackz
    This needs to be a custom class to handle mutation properly.
    This class handles both aten.scatter_ and aten.scatter_reduce_.
    It also handle the case `src` being a scalar properly.
    rr   rs   c              
   C  s   | j d }tjjrddd}||v r|| }| jr%dd | jD \}}}ndd | jD \}}| jd }|||| jd	 ||g| j| j	| j|| 
  d S )
Nr  r`  r_  )r\  multiplyc                 s  r~  r   r  r  rn   rn   ro   r  ?  r  z*ScatterFallback.codegen.<locals>.<genexpr>c                 s  r~  r   r  r  rn   rn   ro   r  A  r  r0   r   )r   rU   r   r  src_is_tensorr  r  Zgenerate_scatter_fallbackr  r  rD  )rA  r  r  Zget_operator_enumr   r   rG  rn   rn   ro   r  6  s$   


zScatterFallback.codegenrh   c                 C  r  r  rn   rD  rn   rn   ro   rV  M  r   zScatterFallback.should_allocatec                 C  r  r  r  rD  rn   rn   ro   r  P  r  z"ScatterFallback.get_mutation_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r  S  rR  z(ScatterFallback.get_unbacked_symbol_defsNTr  include_selfr  r   r  r   r  c          
   
     s   t |t _ jr fdd|||fD }|f}	n fdd||fD }||f}	t jd t| d ||	||dt|ddg|d t	j
|  t	j
  _t	j
  d S )	Nc                   r  rn   r  r  rD  rn   ro   r   e  r   z,ScatterFallback.__init__.<locals>.<listcomp>c                   r  rn   r  r  rD  rn   ro   r   h  r   r  r  r  r  )r  r  r  )ru   ra   r  r  r  r7  r   r  r   rU   r   r>  r  r  r   r  )
rA  r  r   r  r   rG  r  r  tensorsr  r  rD  ro   r  V  s&   
zScatterFallback.__init__r  r  r  )r  r   r  r   r  rh   rr   rs   r  rn   rn   r  ro   r  /  s    


r  c                      r{  )IndexPutFallbackzQ
    This needs to be a custom class to handle mutation and indices properly
    rr   rs   c           	      C  s   dd | j D ^}}}g }t|}t| jD ]\}}| j| d ur)|t| q|tjjj	 q|j
|  |||g|  R   d S )Nc                 s  r~  r   r  r  rn   rn   ro   r    r  z+IndexPutFallback.codegen.<locals>.<genexpr>)r  r  r   r  r  r  rU   r   r  r`  Zgenerate_index_put_fallbackr  r>  )	rA  r  r   ry   valid_indicesr  Ziter_valid_indicesr   r   rn   rn   ro   r    s   zIndexPutFallback.codegenrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   z IndexPutFallback.should_allocatec                 C  r  r  r  rD  rn   rn   ro   r    r  z#IndexPutFallback.get_mutation_namesr   c                 C  r  r   r*   rD  rn   rn   ro   r    rR  z)IndexPutFallback.get_unbacked_symbol_defsc           	   	     s   | _ dd |D } fdd||g|D }d}t jd t| d ||fd||d tj j	d 
  tj  _tj  d S )	Nc                 S  s   g | ]}|d ur|qS r   rn   r   rn   rn   ro   r     r   z-IndexPutFallback.__init__.<locals>.<listcomp>c                   r  rn   r  r  rD  rn   ro   r     r   Zaoti_torch_index_put_outr  zaten.index_put_)r  r  r  r   )r  r  r  r7  r   r  rU   r   r>  r  r  r  r   r  )	rA  r  r   r  ry   
accumulater  r  r  r  rD  ro   r    s    	zIndexPutFallback.__init__r  r  r  r  rn   rn   r  ro   r  z  r  r  c                   @  s"   e Zd Zedd ZdddZdS )	
DeviceCopyc                 C  s   |  stdd | D rtjjs||S tj	| tj	|
  td |f}tt|| | d| |g|S )Nc                 s  s    | ]	}|t jjv V  qd S r   )rU   r   r   r  rn   rn   ro   r    r  z$DeviceCopy.create.<locals>.<genexpr>zDeviceCopy in input programr  )r  r  rN  r1   Zaot_inductorZuse_runtime_constant_foldingr  rU   r   Zadd_device_infor   rI   r  r   r   r   r  )r#  r   r   r  r  rn   rn   ro   r"    s(   

zDeviceCopy.createrr   rs   c                 C  s\   |   }t|dksJ | jr||d | j |d  d S ||d |  |d  d S )Nr   r   r0   )r?  r   r  r  r  rW  rn   rn   ro   r    s   zDeviceCopy.codegenNr  )rj   rk   rl   r8  r"  r  rn   rn   rn   ro   r    s    
r  c                      sJ   e Zd ZdZdddZdddZd fd
dZdddZdddZ  Z	S )r{   z;
    The result of a call to aten._local_scalar_dense.
    rr   r  c                 C  r  r   r*   rD  rn   rn   ro   rM    rR  zDynamicScalar.get_readsrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   zDynamicScalar.should_allocaters   c                   s<   |   t d ttdd| |g || _|| _d S Nr  r  )	r  r  r  r7  r   r   r  symkeypath)rA  r  r  r(  r  rn   ro   r    s   
zDynamicScalar.__init__r   c                 C  s   t | jgS r   )r+   r  rD  rn   rn   ro   r    r   z&DynamicScalar.get_unbacked_symbol_defsc                 C  r`  r   )Zcodegen_dynamic_scalarr  rn   rn   ro   r    r  zDynamicScalar.codegenr  r  r  r  )
rj   rk   rl   r  rM  rV  r  r  r  r:  rn   rn   r  ro   r{     s    


r{   c                      sR   e Zd ZdZdddZdddZd fd
dZdddZdd ZdddZ	  Z
S )r|   z5
    The result of a call to aten._assert_scalar
    rr   r  c                 C  r  r   r*   rD  rn   rn   ro   rM    rR  zAssertScalar.get_readsrh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   zAssertScalar.should_allocaters   c                   s,   t  d ttddg  || _|| _d S r  )r  r  r7  r   r   scalarr(  )rA  r  r(  r  rn   ro   r    s   
zAssertScalar.__init__c                 C  r  r  rn   rD  rn   rn   ro   r    r   zAssertScalar.has_side_effectsc                 C  rb  r   )r%   r  rD  rn   rn   ro   r     r  z%AssertScalar.get_unbacked_symbol_usesc              	   C  s   t jsd S tt|  }tjjr3d| d}tjjj	| j
dd}|d| d| j d| d d S tjjj| j
dd}|d	| d
 |dt| j d ||   d d S )Nzstd::to_string(r   F)r  zif (!(z()) { throw std::runtime_error("Expected z but received " + z); }zif not r  z    raise RuntimeError(z = None)r1   Zscalar_assertsr  r  r  rU   r   r  r  Zcodegen_cpp_sizevarr  r  r(  Zcodegen_python_sizevarr  r  )rA  r  symbolZ
symbol_strZsizevarrn   rn   ro   r    s"   zAssertScalar.codegenr  r  r  )rj   rk   rl   r  rM  rV  r  r  r  r  r:  rn   rn   r  ro   r|     s    


r|   c                   @  s   e Zd ZU ded< ded< dS )ExternKernelNoder   r   zexport_schema.Noder   Nri   rn   rn   rn   ro   r     s   
 r  c                      s   e Zd Z	d"ddd# fddZd#ddZd$d
dZdd Zedd Zdd Z	dd Z
dd Zdd Zd#ddZed%ddZedd Z fd d!Z  ZS )&FallbackKernelNr  rr   rs   c                  sN  t  j|t|t||d d _| _t|tjjtjj	fs,J d| dt
| d| _| _|d u r8i n| _tj j g  _g  _t jtjj	rRd S d j v r[d S  jj}tjj jrs j|d   d S |jrt|std|   j j\}	}d fdd}
tjj ||	|D ]	\}}|
|| qd S )Nr  Fz#Fails to create FallbackKernel for r   z not supportedZ_c10d_functionalr   z'NYI: Can't generate FallbackKernel for rr   rs   c                   s   t  jtjrt |ttfsJ t jrt |ttfrJ |d u r%d S  jd u r,d S d fdd}t	 jrK|d urG|D ]}|| q@d S d S t jsSJ || d S )Nrr   rs   c                   s>   j |    jjrjtt|  d|  d S d S ra  )	alias_namesr  r  
alias_infoZis_writer  r  r7  r   r   )inforA  rn   ro   	add_alias  s   zPFallbackKernel.__init__.<locals>.handle_aliasing_and_mutation.<locals>.add_aliasr  )
ru   r   r   ListTyperv   rw   library_utilsZis_tensor_like_typer  Zis_tensorlist_like_type)r  r  r  Zoptional_tensor_argrD  )r  ro   handle_aliasing_and_mutations  s"   

z=FallbackKernel.__init__.<locals>.handle_aliasing_and_mutationr  )!r  r  rw   use_runtime_dispatchr  ru   r   r  r  r  r   r  r  r   rU   r   Zwarn_fallbackr  r  rY  r   r   _libraryr.  Zmutates_and_returns_first_argr  r  
is_mutabler   rj  r  r  Z
zip_schema)rA  r)  r  r$  nontensor_argsr  r   r  schemar   r  r  r  r  rD  ro   r  '  sN   zFallbackKernel.__init__c                 C  s   | |  | jt| dd S Nr  )(codegen_unbacked_symbol_defs_for_outputsr  r  r   r  rn   rn   ro   codegen_unbacked_symbol_defs  s   z+FallbackKernel.codegen_unbacked_symbol_defsr   c                 C  :   t | dd  }rttjjj|}|d usJ | S t S r  r   r'   rU   r   r   r   r  r+   rA  r  resolvedrn   rn   ro   r       
z'FallbackKernel.get_unbacked_symbol_defsc                   s   t jG dd d  fdd| jD }| || j\}}tjjr=t| j	t
jjr=| ||}dd t| j	jj|D }ndd |D }| j| |S )Nc                   @      e Zd ZU ded< dddZdS )	z)FallbackKernel.codegen_args.<locals>.Shimr
   refrr   r   c                 S  rP  r   )r  rD  rn   rn   ro   r7    rR  z2FallbackKernel.codegen_args.<locals>.Shim.__repr__Nr  )rj   rk   rl   rm   r7  rn   rn   rn   ro   Shim     
 r  c                   s   g | ]} |  qS rn   r  r  r  rn   ro   r     r   z/FallbackKernel.codegen_args.<locals>.<listcomp>c                 S  s"   g | ]\}}t jj||jqS rn   )rU   r   r  r;  r  )r   paramr   rn   rn   ro   r     s    c                 S  r  rn   rA  r  rn   rn   ro   r     r  )r  	dataclassr  r  r  rU   r   r  ru   r  r   r  r  r:  r   r   r  r   r  )rA  r$  r   r   rn   r  ro   r?    s   zFallbackKernel.codegen_argsc                 C  s   | rdd | D }|d S t |tjr|jS t |ttfrItdd |D }dd |D }t|dkr7|d S |D ]}t|j	rD|  S q9|d S d S )Nc                 S  s   g | ]
}|  r|  qS rn   r  r  rn   rn   ro   r     r  z.FallbackKernel.find_device.<locals>.<listcomp>r   c                 s  s    | ]	}t d |V  qd S r   )r  find_devicer  rn   rn   ro   r    r  z-FallbackKernel.find_device.<locals>.<genexpr>c                 S  s   g | ]}|r|qS rn   rn   )r   r   rn   rn   ro   r     r   r0   )
ru   r   r#  r   rv   rw   r+   r   rM   r   )r$  r'  ZdevicesZ
device_setr   rn   rn   ro   r    s$   
zFallbackKernel.find_devicec                 C  s"   t | jtjjr
dS t| j S r  )ru   r  r   r  r  r#   r  rD  rn   rn   ro   r    s   zFallbackKernel.has_side_effectsc                 C  rP  r   )r  rD  rn   rn   ro   r    rR  z+FallbackKernel.get_inputs_that_alias_outputc                 C  s   t | jdks	J | jS r  )r   rY  rD  rn   rn   ro   r    s   z!FallbackKernel.get_mutation_namesc                   sP  t d j ttsJ jj\}	|}fddj
D }j}tjjs9g ||S td d }|||}dd  t|tjjjr]||d |d j}n|jj}t|dkr|jrmjnj}|d j} ||g}	n fddt|jD }	t tjj ||	i d	d
}
tjj !|
 g ||S )Nz4Extern kernel node added for node %s with target %s.c                   s   g | ]}j |fi  qS rn   rq  rz  )r   rA  rn   ro   r     r  z<FallbackKernel.export_extern_kernel_node.<locals>.<listcomp>c                 S  s   t | tjr(|}t |ttfrt|dksJ |d }tjjtj	|
 ddS t | tjrBt |  tjrBtjjdd |D dS tdt|  )	Nr0   r   r   )Z	as_tensorc                 S  s   g | ]
}t j| d qS )r   )export_schemaTensorArgumentr  )r   r   rn   rn   ro   r     s    zZFallbackKernel.export_extern_kernel_node.<locals>.handle_single_output.<locals>.<listcomp>)Z
as_tensorszUnsupported return type )ru   r   Z
TensorTyperv   rw   r   r  ZArgumentr"  r  r  r  getElementTypeRuntimeErrorr   )return_typeoutputr   rn   rn   ro   handle_single_output  s"   
zFFallbackKernel.export_extern_kernel_node.<locals>.handle_single_outputr   r0   c                   s   g | ]
\}} |j |qS rn   r  )r   Zreturn_schemar  )r  rn   ro   r     s    
)r  r  r  metadata)r   r   )"r  r  r  r  ru   r  r  r  r  r:  r  rU   r   Zaot_moder   Zserialize_inputsr   Z_higher_order_opsZ	torchbindZCallTorchBindr  returnsr   r   r  r  r  r   r  r  rV   r   Zextern_kernel_nodesr  )rA  r   Zordered_kwargsr  
serializerZnamed_argumentsr  r  r  Zoutput_argumentsr   rn   )r  r   rA  ro   export_extern_kernel_node  sL   




z(FallbackKernel.export_extern_kernel_nodec                   s:  j }|jdkr+t|tjjsJ tjjr*ddl	m
} t||vr*td| d_n|jdkr:t|tjjs9J ntjjrAd_dfd	d
}ddd  jrZ|  n<g   }tjjrt|tjjrt fddt||jjD r|  ntjj| tjtr  d S )Nr	  r   )inductor_fallback_opszG%s is missing a c-shim implementation, using proxy executor as fallbackTZ
_quantizedrr   rs   c               	     sD   d }    }   j j|  j| jr j d S  j d S r   )r  Z,generate_fallback_kernel_with_runtime_lookupr  r  r  r  r  r  )r   Zexported_argsr  rn   ro   do_runtime_dispatchF  s   
z3FallbackKernel.codegen.<locals>.do_runtime_dispatchr   torch.JitTyperh   c                 S  s(   t | tjpt | tjot |  tjS r   )ru   r   Z
NumberTypeZOptionalTyper  r  rn   rn   ro   	is_numberU  s   z)FallbackKernel.codegen.<locals>.is_numberc                 3  s&    | ]\}}d |v o |j V  qdS )zc10::complexNr  )r   Zarg_strZop_arg)r  rn   ro   r  c  s
    
z)FallbackKernel.codegen.<locals>.<genexpr>r  )r   r  rr   rh   )r  r  ru   r   r  r  rU   r   r  Ztorchgen.aoti.fallback_opsr  r   r  r  r  r  r?  rD  r]  r   r   r  r  Zgenerate_fallback_kernelr)  rm  rE  r  )rA  r  r  r  r  r   rn   )r  rA  r  ro   r  /  sF   




zFallbackKernel.codegenr  r   c                 C  s"   t | j| jt|  t|  S r   )r+  r   r   rG   r   r   )r  rn   rn   ro   tensor_to_layoutu  s   

zFallbackKernel.tensor_to_layoutc                   s   t jf}||vrtjjnt }|  j|g|R i |\}}}}	}
W d    n1 s-w   Y   ||}|d u rJ t|d||||	|
dn|sPJ d t	|d||||	|
d fdd|g }t
|tttfrw|_|S |g_|S )Nr  r  z"Not sure where to find device infoc                   s   t ttfrt fddttD S t tr, fdd D S t tj	r;t
 S t trBS t tjrLjjS d u sZJ dt dd S )Nc                 3  s,    | ]} | t |fg V  qd S r   r   r   generate_outputr  r  rn   ro   r    s
    
zAFallbackKernel.create.<locals>.generate_output.<locals>.<genexpr>c                   s*   i | ]\}}| |t |fg qS rn   r  )r   r  r  r  rn   ro   r     r*  zBFallbackKernel.create.<locals>.generate_output.<locals>.<dictcomp>zFallbackKernel output type z is not supported)ru   rv   rw   r   r   r   rx   ru  r   r#  MultiOutputr  r   SymIntr   r  )r  r  r#  r  packed)r  r  ro   r    s,   



z.FallbackKernel.create.<locals>.generate_output)r	  Z*_fused_moving_avg_obs_fq_helper_functionalrU   r   r"  r   r)  r  r7  r  ru   rv   rw   rx   r  )r#  r  r   r   Zfake_incorrect_kernelscontextr'  r$  r%  r  r  r   r  rn   r  ro   r"  ~  sL   	
	
zFallbackKernel.createc                   s
   t   S r   )r  r  rD  r  rn   ro   r    r  zFallbackKernel.apply_constraintr   r  r  )r  r   )rj   rk   rl   r  r  r  r?  r  r  r  r  r  r  r  r  r8  r"  r  r:  rn   rn   r  ro   r  &  s(    	
l




OF
Fr  c                      s<   e Zd ZdZdddZdddZd	d
d fddZ  ZS )ComplexViewz9View a complex number as two dtyped numbers or vice versarr   rh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   zComplexView.should_allocaterX  c                 C  r  r  r  rD  rn   rn   ro   r    r4  z(ComplexView.get_inputs_that_alias_outputNr  rs   c                  s   t  j||||||d d S )Nr  )r  r  )rA  r)  r  r$  r  r  r  r  rn   ro   r    s   

zComplexView.__init__r  r  r  )rj   rk   rl   r  rV  r  r  r:  rn   rn   r  ro   r    s    

r  c                   @  r  )	r  r  r   rr   r  c                 C  rP  r   r  rD  rn   rn   ro   r     rR  zMultiOutputLayout.get_deviceNr  )rj   rk   rl   rm   r   rn   rn   rn   ro   r    r  r  c                      sN   e Zd Zdd ZdddZd fddZdddZdddZdddZ  Z	S )r  c                 C  s   t |dkrW|d \}}t|tr!| | d| d|dd  S t|tr=tjj|| 	 t
|}| ||dd  S t|trR| | d| d|dd  S td||S )Nr   []r0   z['z']znon supported index type: )r   r  rv   codegen_list_tuple_accessrw   rU   r   r  Zcodegen_tuple_accessr  r   rx   r  )rA  basenamer  Zityper   Ztuple_accessrn   rn   ro   r    s   
 

 
z%MultiOutput.codegen_list_tuple_accessrr   rs   c                 C  s2   | |  | | jd  | j | | d S r  )Zcodegen_multi_outputr  r  r  r  rE  r  rn   rn   ro   r    s
   zMultiOutput.codegenr)  r   r  list[tuple[Any, ...]]c                   s8   t  d ||gd tj| | _tj|  || _d S r  )r  r  rU   r   r  r   r  r  )rA  r)  r  r  r  rn   ro   r    s   
zMultiOutput.__init__r   c                 C  s   | j d  S r  )r  r  rD  rn   rn   ro   r    r  z$MultiOutput.get_unbacked_symbol_usesrh   c                 C  s&   t | jdkrt| jd trdS dS )Nr0   r   TF)r   r  ru   r  rD  rn   rn   ro   rV    s
   zMultiOutput.should_allocaterX  c                 C  s   dd | j D S )Nc                 S  s.   g | ]}t |trt| d kr| qS r   )ru   r  r   r  r  r   r  rn   rn   ro   r   #  s    z<MultiOutput.get_inputs_that_alias_output.<locals>.<listcomp>)r  rD  rn   rn   ro   r  "  s   z(MultiOutput.get_inputs_that_alias_outputr  )r)  r   r  r  rr   rs   r  r  r  )
rj   rk   rl   r  r  r  r  rV  r  r:  rn   rn   r  ro   r    s    


r  c                   @  s  e Zd ZU dZded< dtddZdud	d
ZdvddZdwddZdxddZ	dyddZ
dzd{ddZd|dd Zd}d!d"Zd~d#d$Zd}d%d&Z	'ddd+d,Zdd-d.Zdd1d2Z	'ddd4d5Zdd7d8Zdd:d;Zdd<d=Zdd?d@ZddBdCZddEdFZdtdGdHZdtdIdJZddMdNZddPdQZdydRdSZddTdUZddVdWZ ddYdZZ!dd\d]Z"dd_d`Z#dzddcddZ$e%ddfdgZ&ddidjZ'ddkdlZ(ddndoZ)e%dpdq Z*dydrdsZ+e+Z,dS )r@  zC
    TensorBox / StorageBox allow in-place mutation of Tensors
    rb   r(  rr   rh   c                 C  rz  r   r  rD  rn   rn   ro   r  5  r  z!MutableBox.has_exceeded_max_readsr  c                 C  rz  r   r  rD  rn   rn   ro   r   8  r  zMutableBox.get_devicer  c                 C  rz  r   r  rD  rn   rn   ro   r  ;  r  zMutableBox.make_loaderr  c                 C  rz  r   )r(  r  rD  rn   rn   ro   r  >  r  zMutableBox.make_indexerr  c                 C  rz  r   )r(  r  rD  rn   rn   ro   r  A  r  zMutableBox.get_strider   c                 C  rz  r   r  rD  rn   rn   ro   r  D  r  zMutableBox.get_nameNr  r  c                 C  r  r   )r(  r  r  rn   rn   ro   r  G  r   zMutableBox.has_large_inner_fnr  r   rs   c                 C  r  r   r  r  rn   rn   ro   r  J  r   zMutableBox.mark_reusec                 C  rz  r   r  rD  rn   rn   ro   r  M  r  zMutableBox.realize_hintc                 C  rz  r   )r(  r  rD  rn   rn   ro   r  P  r  zMutableBox.unwrap_viewc                 C  rz  r   )r(  r  rD  rn   rn   ro   r  S  r  zMutableBox.freeze_layoutFr   r  r  c                 C     | j ||S r   )r(  r  r  rn   rn   ro   r  V  r  z*MutableBox.freeze_layout_with_stride_orderc                 C  r  r   )r(  r  r  rn   rn   ro   r  [  r   z(MutableBox.freeze_layout_with_fill_orderr   r  c                 C  r  r   )r(  r  r  rn   rn   ro   r  ^  r   z(MutableBox.freeze_layout_with_same_orderr  c                 C  r  r   )r(  r  r  rn   rn   ro   r  a  r  z+MutableBox.freeze_layout_with_exact_stridesr  c                 C  rz  r   )r(  r  rD  rn   rn   ro   r  f  r  zMutableBox.get_read_writesr  c                 C  rz  r   re  rD  rn   rn   ro   rM  i  r  zMutableBox.get_readsc                 C  rz  r   rd  rD  rn   rn   ro   r  l  r  zMutableBox.num_readsr]   c                 C  rz  r   r  rD  rn   rn   ro   r  o  r  zMutableBox.get_storage_numelr   c                 C  rz  r   r  rD  rn   rn   ro   r  r  r  zMutableBox.get_reduction_typer  c                 C  rz  r   r  rD  rn   rn   ro   r  u  r  zMutableBox.get_reduction_sizec                 C  rz  r   r  rD  rn   rn   ro   r  x  r  zMutableBox.is_externc                 C  rz  r   )r(  r  rD  rn   rn   ro   r  {  r  zMutableBox.is_no_opr   r  c                 C  r  r   r  r  rn   rn   ro   r  ~  r   zMutableBox.constant_to_devicerX  c                 C  rz  r   )r(  r  rD  rn   rn   ro   r    r  zMutableBox.get_mutation_namesc                 C  rz  r   )r(  r  rD  rn   rn   ro   r    r  zMutableBox.get_operation_namec                 C  rz  r   )r(  r  rD  rn   rn   ro   r    r  z'MutableBox.get_inputs_that_alias_outputc                 C  rz  r   r  rD  rn   rn   ro   r    r  zMutableBox.realizer   c                 C  rz  r   r{  rD  rn   rn   ro   r    r  z#MutableBox.get_unbacked_symbol_usesrF  c                 C  rz  r   r  rD  rn   rn   ro   rN    r  zMutableBox.get_read_namesrU  c                 C  rz  r   )r(  rV  rD  rn   rn   ro   rV    r  zMutableBox.get_defining_opr  r  c                 C  r  r   )r(  r  r  rn   rn   ro   r    r   zMutableBox.codegen_referencer   c                 C  rz  r   r(  rt  rD  rn   rn   ro   r)    s   
zMutableBox.layoutrm  c                 C  rz  r   r  rD  rn   rn   ro   r     r  zMutableBox.get_layoutc                 C  rz  r   r  rD  rn   rn   ro   rt    r  zMutableBox.get_output_specry  c                 C  rz  r   r  rD  rn   rn   ro   r     r  zMutableBox.get_sizec                 C  r  r   )r(  r   rD  rn   rn   ro   r     r~  zMutableBox.dtypec                 C  sn   t | jtrt| j dt| jj d}d}| jj}nt| j d}| j}d}|tt||g}d|S )Nr`  z))r   
)ru   r(  r@  r   rj   rb  r   rc  )rA  Zline0Zendlr  r]  rn   rn   ro   r    s   


zMutableBox.__str__r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )-rj   rk   rl   r  rm   r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  rM  r  r  r  r  r  r  r  r  r  r  r  r  rN  rV  r  r  r)  r   rt  r   r   r  r7  rn   rn   rn   ro   r@  -  s\   
 
































r@  c                   @  s   e Zd Zedd ZdS )ra   c                 C  s   t | tr| S tt| S r   )ru   r   ra   rt  )r(  rn   rn   ro   r"    s   
zTensorBox.createN)rj   rk   rl   r  r"  rn   rn   rn   ro   ra     s    c                   @  sT   e Zd Zdd Zdd ZdddZdd
dZdddZdd ZdddZ	dd Z
dS )rt  c                 C  s&   t | jttfr| j tjjv S dS r  )ru   r(  rZ  r-  r  rU   r   graph_inputsrD  rn   rn   ro   r    s   zStorageBox.is_input_bufferc                 C  s   t | jto| j tjjv S r   )ru   r(  rQ  r  rU   r   r   rD  rn   rn   ro   r    s   zStorageBox.is_module_bufferrr   r   c                 C  s   t | jtttttfr| j S t | jtt	t
tfs!J t| j| j }| j }td t| j | j | j d| jd| _tj| j| j_tj| j | j| j_|| j_|| j_| jjS )Nr  r  )ru   r(  r  r  rZ  r-  r  r  rA  r|  r1  r_  r   rT  rQ  r   r   r   r   rU   r   r  r   r  r6  r:  r8  )rA  r:  r8  rn   rn   ro   r    s<   



	
zStorageBox.realizers   c                 C  s4   t | jttfr| j jdkr|   dS dS dS )zL
        Called on buffers we expect to be forced to realize later.
        r0   N)ru   r(  rA  r|  r0  Znontrivial_read_countr  rD  rn   rn   ro   r    s   zStorageBox.realize_hintrh   c                 C  s"   t | jto|  tjkp|  S r   )ru   r(  rA  r  r1   Zrealize_acc_reads_thresholdr  rD  rn   rn   ro   r    s   z!StorageBox.has_exceeded_max_readsc                   sh   |dkr2t | jttfr2t| jr'| j  ddg}t fdd|D r'dS |  tj	kp1| 
 S dS )zj
        A heuristic to decide if we should realize a tensor
        that is used multiple times.
        r0   expZsigmoidc                 3  s    | ]}| j v V  qd S r   )Zused_opsr  Zopcountrn   ro   r    r  z5StorageBox.should_realize_on_reuse.<locals>.<genexpr>TF)ru   r(  rA  r|  r
  r0  r]  r  r1   Zrealize_reads_thresholdr  )rA  r  Z	heavy_opsrn   r  ro   should_realize_on_reuse  s   

z"StorageBox.should_realize_on_reuser  r   c                 C  s   |  |r|   d S d S r   )r  r  r  rn   rn   ro   r    s   
zStorageBox.mark_reusec                 C  rz  r   rd  rD  rn   rn   ro   r    r  zStorageBox.num_readsNr  r  r  r  )rj   rk   rl   r  r  r  r  r  r  r  r  rn   rn   rn   ro   rt    s    

!


rt  c                   @  s*   e Zd ZU ded< ded< dZded< dS )Subgraphr   r   ztorch.fx.GraphModulegraph_moduleNzOptional[GraphLowering]r   )rj   rk   rl   rm   r   rn   rn   rn   ro   r    s   
 r  buffersr  c                 C  s,   dd | D } t tdd | D t | k S )Nc                 S  s"   g | ]}t |tr| n|qS rn   )ru   r-  r  r   rv  rn   rn   ro   r   &  s    z(_has_aliased_buffers.<locals>.<listcomp>c                 s  r
  r   )r_  r  rn   rn   ro   r  +  r  z'_has_aliased_buffers.<locals>.<genexpr>)r   r+   )r  rn   rn   ro   _has_aliased_buffers%  s   r  c                      s\   e Zd ZU dZded< dZded< dZded< d fddZedddZ	dddZ
  ZS )InvokeSubgraphNOptional[Subgraph]subgraphzOptional[list[TensorBox]]operandsOptional[list[MultiOutput]]r  r  list[TensorBox]r)  r  rr   rs   c                   s6   t  jd ||d || _tj| | _tj|  d S r  )r  r  r  rU   r   r  r   r  )rA  r  r  r)  r  rn   ro   r  4  s   zInvokeSubgraph.__init__c                   s\  t jjjd }dd |D } fdd|D }dd }g }t|D ] \}}t|tr0|| q!|||  }	| 	||	 q!|}|jd u rqt jj
|j||jd|_t |j |jj|  W d    n1 slw   Y  |jj}
d }|D ]}t|ts| } nqy|d usJ t||t|dd	dfddfddt|
D }
|
_|
S )Nr  c                 S     g | ]}|j d  qS r  r
  r  rn   rn   ro   r   D  r   z)InvokeSubgraph.create.<locals>.<listcomp>c                   r  rn   r  r  r  rn   ro   r   I  r   c                 S  s   dd | D S )Nc                 S  s$   g | ]}t |tjr|jjn|qS rn   )ru   r   r  r   r  r   rn   rn   ro   r   L  r  zBInvokeSubgraph.create.<locals>.handle_sym_expr.<locals>.<listcomp>rn   r  rn   rn   ro   handle_sym_exprK  r  z.InvokeSubgraph.create.<locals>.handle_sym_exprZgmZexample_inputsZsubgraph_namer  )r  r  r)  r  rb   indr   c                   sH   t | ttfr	| S tt|  |  |  |  | 	 j
d t|fgS )Nr  )ru   r   r_  r  r+  r   r   r   r  r   r,  rv   )r  r  )invoke_subgraphrn   ro   create_outputs  s   z,InvokeSubgraph.create.<locals>.create_outputc                   s   g | ]	\}} ||qS rn   rn   r   r   r  )r  rn   ro   r     r  )r  rb   r  r   )rU   r   r  r   r   ru   r   r  r   r  make_subgraphr  r   set_graph_handlerrungraph_outputsr   r  r  r  )r#  r  r  fx_operandsfake_operandsr  Znew_operandsr   operandZexample_strider  r   rn   )r#  r  r  ro   r"  @  sJ   


zInvokeSubgraph.createc                 C  r`  r   )Zcodegen_invoke_subgraphr  rn   rn   ro   r    r  zInvokeSubgraph.codegen)r  r  r  r  r)  r  rr   rs   )r  r  r  )rj   rk   rl   r  rm   r  r  r  r8  r"  r  r:  rn   rn   r  ro   r  .  s   
 Fr  c                      s~   e Zd ZU dZded< dZded< dZded< dZded< dZd	ed
< d  fddZ	e
d!ddZd"ddZd#ddZ  ZS )$ConditionalNr   	predicate7Optional[list[Union[TensorBox, ShapeAsConstantBuffer]]]r  r  true_subgraphfalse_subgraphr  r  rb   -list[Union[TensorBox, ShapeAsConstantBuffer]]r  r)  r  r  ,Optional[dict[sympy.Symbol, pytree.KeyPath]]rr   rs   c           	        sj   || _ || _|| _|| _t|g| \}}t jd |||d |d ur&|| _tj	
| | _tj	|  d S N)r   r)  r  r  )r  r  r  r  _split_by_sym_typer  r  r  rU   r   r  r   r  )	rA  r  r  r  r  r)  r  sym_argsr$  r  rn   ro   r    s   	zConditional.__init__ra   true_fnfalse_fnc              	     s   |}fdd|D }tjjjd }dd |D }||fD ]/}|jd u rOtjj|j||jd|_t|j |jj	|  W d    n1 sJw   Y  q |jj
}|jj
}	d|fd|	ffD ]\}
}t|rrtd|
 d	| q`t|t|	ksJ ||	ftt||	D ]5\}\}}| | ksJ |||f| | ksJ |||f| j| jksJ |||fqtd
d |g| D }ttjjjtjjjdd }|d usJ dt||||t|d|dddd  fddtt|tjjjd D }|_|S )Nc                   r  rn   r  r  r  rn   ro   r     r   z&Conditional.create.<locals>.<listcomp>r  c                 S  r  r  r  r  rn   rn   ro   r     r   r  r  r  zVOutput aliasing is currently not supported in compiled torch.cond. The outputs of the z% subgraph of torch.cond are aliased: c                 s  s"    | ]}t |ts| V  qd S r   )ru   r   r   )r   orn   rn   ro   r    s    
z%Conditional.create.<locals>.<genexpr>r  zcannot determine devicer  )r  r  r  r  r)  r  r   Union[int, torch.SymInt]rr   Union[int, sympy.expr]c                 S  s   t | tr| S | jjS r   )ru   r   r   r  )r   rn   rn   ro   _maybe_expr  s   
z'Conditional.create.<locals>._maybe_exprc              
     sf   g | ]/\}\}}t t| |  fd d| D  fdd| D | jdt|fgqS )c                   r   rn   rn   r   r  r  rn   ro   r     r   z1Conditional.create.<locals>.<listcomp>.<listcomp>c                   r   rn   rn   r  r  rn   ro   r     r   r  )	r  r+  r   r   r   r   r   r,  rv   )r   r   r  Zmerged_output)r  conditionalrn   ro   r     s    
r  )r   r  rr   r  )r  rU   r   r  r   r  r  r   r  r  r  r  r  r   r   r   r   r   r   r,  r  r'   r   r   r
  r  r  r  r  )r#  r  r  r  r  r  r   r  Ztrue_outputsZfalse_outputsr   r  r   tofor   r  rn   )r  r#  r  ro   r"    sp   

$
	zConditional.createc                 C  s*   | |  ||  | jt| di  d S r  )Zcodegen_conditionalr  r  r  r   r  rn   rn   ro   r    s   
zConditional.codegenr   c                 C  r  r  r  r  rn   rn   ro   r    r  z$Conditional.get_unbacked_symbol_defs)r  rb   r  r  r  r  r  r  r)  r  r  r  rr   rs   )r  ra   r  r  r  r  r  r  r  r  )rj   rk   rl   r  rm   r  r  r  r  r  r8  r"  r  r  r:  rn   rn   r  ro   r    s   
 
Vr  r   	list[Any]-tuple[list[ShapeAsConstantBuffer], list[Any]]c                 C  s<   g }g }| D ]}t |tr||j q|| q||fS r   )ru   r   r  r  )r   Znon_sym_argsr  r  rn   rn   ro   r
    s   
r
  c                      st   e Zd ZU dZded< dZded< dZded< dZded< dZded	< d fddZ	e
dddZdddZ  ZS )	WhileLoopNr  carried_inputsadditional_inputsr  cond_subgraphbody_subgraphr  r  r  r  r)  r  rr   rs   c                   sZ   || _ || _|| _|| _t|| \}}t jd |||d tj	| | _
tj|  d S r	  )r  r  r  r  r
  r  r  rU   r   r  r   r  )rA  r  r  r  r  r)  r  r$  r  rn   ro   r  -  s   zWhileLoop.__init__cond_fnbody_fnc              	     s   fdd|D } fdd|D }|| }t jjjd t jjjd  }dd |D }||fD ]/}|jd u r^t jj|j||jd|_t |j |jj|  W d    n1 sYw   Y  q/|jj	}	|jj	}
t
|
rrtd|
 t|	d	ks|J |	|	d
 }t|ts| tjksJ |t| d
ksJ |t|d
ksJ d|d
  }t|t|
ksJ ||
ftt||
D ]N\}\}}ddd}|| |  || |  | | ksJ ||||f| | ksJ |||f| j| jksJ |||fqt||||t|ddfddt|
D }t||D ]\}}| t jjv rBt jj|  q,|_|S )Nc                   r  rn   r  r  r  rn   ro   r   M  r   z$WhileLoop.create.<locals>.<listcomp>c                   r  rn   r  r  r  rn   ro   r   N  r   r  c                 S  r  r  r  r  rn   rn   ro   r   R  r   r  zOutput aliasing is currently not supported in compiled torch.while_loop. The outputs of the body_fn subgraph of torch.while_loop are aliased: r0   r   z9torch.while_loop is assumed to have at least one operand.	lhs_exprslist[Union[int, sympy.expr]]	rhs_exprsrr   rs   c                 S  s(   t | |D ]\}}tjj|| qd S r   )r   rU   r   r   r  )r!  r#  lhsrhsrn   rn   ro   _guard_list_equalsy  s   z,WhileLoop.create.<locals>._guard_list_equalsr  )r  r  r  r  r)  c              
     sF   g | ]\}}t t| | | | | jd  t|fgqS )r  )	r  r+  r   r   r   r  r   r,  rv   r  )
while_looprn   ro   r     s    )r!  r"  r#  r"  rr   rs   ) rU   r   r  r   r  r  r   r  r  r  r  r  r   ru   r   r   r   rh   r   r   r   r   r  r   r,  r  r  r  r  r  r\  r  )r#  r  r  r  r  Z
all_inputsZfx_all_inputsZfake_all_inputsr  Zcond_outputsZbody_outputsr  r   r   opZbor&  r  r  r   rn   )r#  r'  ro   r"  E  sr   


 &
	zWhileLoop.createc                 C  r`  r   )Zcodegen_while_loopr  rn   rn   ro   r    r  zWhileLoop.codegen)r  r  r  r  r  r  r  r  r)  r  rr   rs   )r  r  r  r  r  r  r  r  r  )rj   rk   rl   r  rm   r  r  r  r  r  r8  r"  r  r:  rn   rn   r  ro   r  %  s   
 fr  c                      s@   e Zd Z	dddd fddZd fdd	ZdddZ  ZS )r   Nr  rr   rs   c             	     s~   t  j|||||d |d ddlm} dd |D }	||g ||	R |}
|
d us+J |
| _tjj|
d | _	| tjj|
< d S )N)r   r  r   )get_effect_keyc                 S  s    g | ]}t |tr|jn|qS rn   )ru   r  r   )r   ri  rn   rn   ro   r     s    z,EffectfulKernel.__init__.<locals>.<listcomp>)
r  r  Ztorch._higher_order_ops.effectsr)  effect_typerU   r   Zeffectful_opsr  prev_effect_buffer)rA  r)  r  r$  r  r  r   r  r)  Zuncovered_argsr*  r  rn   ro   r    s$   
zEffectfulKernel.__init__r  c                   s0   t   }| jd ur|jt| j  |S r   )r  r  r+  r  r\  r2   ri  r  )rA  r  r  rn   ro   r    s   

zEffectfulKernel.get_read_writesrh   c                 C  r  r  rn   rD  rn   rn   ro   r    r   z EffectfulKernel.has_side_effectsr   r  r  r  )rj   rk   rl   r  r  r  r:  rn   rn   r  ro   r     s    	 
r   c                   @  r]  )r,  Nr^  rn   rn   rn   ro   r,    s    r,  c                   @  s\   e Zd ZU ddlmZ ded< ded< dd ZddddZdddZdddZ	dddZ
d	S )r  r   )FakeScriptObjectr   r   +Union[FakeScriptObject, torch.ScriptObject]r   c                 C  rP  r   r   rD  rn   rn   ro   r    rR  zTorchBindObject.get_nameNr  r  rr   c                 C  rP  r   r   r  rn   rn   ro   r    rR  z!TorchBindObject.codegen_referencec                 C  rP  r   r   rD  rn   rn   ro   	get_value  rR  zTorchBindObject.get_valuetorch.ScriptObjectc                 C  s   t | jtjr
| jS | jjS r   )ru   r   r   ZScriptObjectZreal_objrD  rn   rn   ro   r!    s   zTorchBindObject.get_real_objr   c                 C  sB   |   }t| }t|d }dd |D }tdd |dS )Nr   c                 S  s(   g | ]}t |tjr| |  qS rn   )ru   r   r#  r^  Znumelr  rn   rn   ro   r     s    
z1TorchBindObject.get_buf_bytes.<locals>.<listcomp>c                 S  s   | | S r   rn   )r   yrn   rn   ro   r    s    z/TorchBindObject.get_buf_bytes.<locals>.<lambda>)r!  rx   Z__obj_flatten__r  r  r  r  )rA  Zreal_script_objZ	flat_dictZ
flat_elemsZ
flat_sizesrn   rn   ro   get_buf_bytes  s   zTorchBindObject.get_buf_bytesr   r  )rr   r-  )rr   r/  r  )rj   rk   rl   Z"torch._library.fake_class_registryr,  rm   r  r  r.  r!  r1  rn   rn   rn   ro   r    s   
 

r  c                   @  s2   e Zd ZU ded< ded< dd ZddddZdS )r  r   r   r  r   c                 C  rP  r   r   rD  rn   rn   ro   r    rR  zGeneratorState.get_nameNr  r  rr   c                 C  rP  r   r   r  rn   rn   ro   r    rR  z GeneratorState.codegen_referencer   r  )rj   rk   rl   rm   r  r  rn   rn   rn   ro   r    s
   
 r  c                   @  sH   e Zd ZdddZdddZddddZedddZedddZdS )_CollectiveKernelrr   rh   c                 C  r  r  rn   rD  rn   rn   ro   rV    r   z!_CollectiveKernel.should_allocatec                 C  r  r  rn   rD  rn   rn   ro   r    r   z"_CollectiveKernel.has_side_effectsNr  r   rs   c                 C  sB   t | jtjju sJ d| j}|jj| _dd |jjD | _	d S )Nz,Setting cpp kernel needs a valid op_overloadc                 S  r  rn   r  r  rn   rn   ro   r   "  r  z9_CollectiveKernel.set_cpp_kernel_name.<locals>.<listcomp>)
r   r  r   r  r  r   r   r  r  r  )rA  r  r  rn   rn   ro   r    s   
z%_CollectiveKernel.set_cpp_kernel_namer  !Union[TensorBox, list[TensorBox]]c                   s  t jj | j||g|R i |\}}}}}	W d    n1 s!w   Y  |	r1J | d|	 |D ]}
|
  q3|d   | t d||||t|}j	
 fdd|D  j
dd |D  d|v rj	tt d|d  j|d   d S d S )Nr  r   r  c                   r  r  r  r  r   r  rn   ro   r   G  rl  z4_CollectiveKernel.create_inplace.<locals>.<listcomp>c                 S  s   g | ]}|  qS rn   rS  r  rn   rn   ro   r   K  r   r   )rU   r   r"  r)  r  r   r7  r  Ztree_leavesr  r  r  r  r  r  )r#  r  r  r   r   _example_outputr$  r%  r  r  
tensor_argZinpsrn   r4  ro   create_inplace,  s>   


z _CollectiveKernel.create_inplacec                   s   t jj  j||g|R i |\}}}}}	W d    n1 s!w   Y  |	r1J | d|	 |D ]}
|
  q3t|tr` ||} t|d|||| fddt	|D _
j
S   |||||g_
S )Nr  r  c                   s(   g | ]\}}t  |t|fgqS rn   )r  r  rv   )r   r   r  r#  r  rn   ro   r     s    z9_CollectiveKernel.create_out_of_place.<locals>.<listcomp>)rU   r   r"  r)  r  ru   rv   r  r  r   r  r  )r#  r  r  r   r   r'  r$  r%  r  r  r6  r   rn   r8  ro   create_out_of_placei  sD   


z%_CollectiveKernel.create_out_of_placer  r   rP  )r  r3  rr   rs   )r  r3  )	rj   rk   rl   rV  r  r  r8  r7  r9  rn   rn   rn   ro   r2    s    

<r2  c                      s4   e Zd Zdd ZedddZd fd
dZ  ZS )_WaitKernelc                 C  s`   | j d }t|tr|j d gS t|tr.|j d }t|tr,|jd \}}|j | gS g S g S r  )r  ru   r2  r  r  )rA  r  Zcollr   r   rn   rn   ro   get_volatile_reads  s   




z_WaitKernel.get_volatile_readsr  ra   rr   rs   c           	      C  s   t jj | ||\}}}}}W d    n1 sw   Y  |r*J | d| | t| d||||}|jtt| d|| d S )Nr  r  )	rU   r   r"  r)  r7  r   r  r  r  )	r#  r  r  r5  r$  r%  r  r  r  rn   rn   ro   create_wait  s(   

z_WaitKernel.create_waitr  c                   s6   t   }|  }|D ]}|jt|  q|S r   )r  r  r;  r  r\  r2   ri  r  )rA  r  Zvolatile_readsZvrr  rn   ro   r    s
   
z_WaitKernel.get_read_writes)r  ra   rr   rs   r  )rj   rk   rl   r;  r8  r<  r  r:  rn   rn   r  ro   r:    s
    r:  r   r   r  c                 C  sd   t | ttfrt| S t | ttfr%ttj  }| D ]}|t	|O }q|S t | t
jr/t| S t S r   )ru   r)   r   r%   rw   rv   r+   r}   r   rL  r   r#  )r   r%  r   rn   rn   ro   rL    s   rL  )rp   rq   rr   rs   )r   r   rr   r   )r   r   rr   r   )r   r   r   r   rr   r   r   )r   r   r   r   rr   r   )r   r   rr   r   r  )r   r   r   rh   rr   rs   )r   rb   r   rh   rr   r   )r   r   r   rh   rr   r   )r   r   rr   r   )r   r   rr   r   )r   r  rr   rh   )r   r  r  r   rr   rh   )r  r  r  r  r  r  rr   rh   )r  r   r!  r"  rr   r   )r   r;  r   rf  rr   rT   )rc  r   r   rf  rd  rh   rr   re  )r   rb   rr   rh   )TFNFN)r   rb   rg  rh   rn  rh   ro  rp  r  rh   r  rp  rr   rq  )r   rb   ro  r   rr   rh   )r   r  r  r  rr   rh   )r   rf  rr   r   )r  r  rr   rh   )r   r  rr   r  )r   r   rr   r  (  
__future__r   r  r  r  r  loggingtextwrapr8  r  collections.abcr   r   r   r   r   enumr   r	   r
   r   r   r   r   r   r   r   r   Ztyping_extensionsr   r   r   Zunittest.mockr   r}   r   r   r   Ztorch._export.serde.schemaZ_exportZserder  r  Ztorch._library.utilsr  r.  r  Ztorch._loggingr   Ztorch.fxZtorch.utils._pytreeZ_pytreer  Ztorch._dynamo.utilsr   Ztorch._export.serde.serializer   Z*torch._higher_order_ops.auto_functionalizer   Ztorch._inductorr   Ztorch._prims_commonr   r   r    r!   r"   Ztorch._subclasses.fake_tensorr#   Z%torch.fx.experimental.symbolic_shapesr$   r%   r&   r'   r(   r)   Ztorch.utils._ordered_setr+   Ztorch.utils._sympy.functionsr,   r-   r.   Ztorch.utils._sympy.symbolr/   rY  r1   r2   Zcodegen.commonr3   r4   r5   r6   r7   r8   r9   r:   Z	loop_bodyr;   Zops_handlerr<   r=   r>   r?   Zruntime.benchmarkingr@   Zruntime.hintsrA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   ZvirtualizedrS   rT   rU   Ztorch.fx.noderV   Zcodegen.cuda.cuda_templaterW   r   rX   rY   r   rm   r  __version__rZ  rY  ImportErrorrZ   r[   r\   r   r]   r?  r^   	getLoggerrj   r  rb  r	  rx   r   rc   r  re   r   r   r   r   r   r  r  r   r   r   r   r   r  r	  r
  r  r  r2  rb   r  r  r@  rA  rT  rb  r{  r|  ZINNER_FN_TYr  r  r#  r1  r[  r_  r   rm  r*  rx  ry  rz   r  r  r  r  r-  r  r  r  r  r  r  r  r   rm  r+  r   r)  r1  r3  r7  r<  ru  rW  rZ  r\  rQ  r_  r   r  r  r  rh   rv   ZPrimitiveInfoTyper  r  r  r  r  r  r  r  r  rQ  rR  r  r  rZ  ra  r|  r  r  r  r  r  r  r  r{   r|   r  r  r  r  r  r@  ra   rt  r  r  r  r  r
  r  r   r,  r  r  r2  r:  rL  rn   rn   rn   ro   <module>   s8   , H
*

"	


& tD  #?     7:& _ E <
_N+: S(R L 'T{
  JE"40GA ,      5-J 0*0K/$9   &= W	\  /" 8