o ZhE@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddlm Z ddl m Z mZmZmZmZmZmZddlmZddlZddlZddlZddlmZddlmZddlmZmZm Z dd l!m"Z"m#Z#m$Z$m%Z%d d l&m'Z'd d l(m)Z)m*Z*m+Z+d dl,m-Z-d dl.m/Z/d dl0m1Z1m2Z2m3Z3d dl*m4Z4m5Z5d dl6m7Z7d dl8m9Z9m:Z:d dl+m;Z;mm?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZId dlJmKZKmLZLmMZMddlNmOZOddlPmQZQmRZRmSZSmTZTddlUmVZVddlWmXZXmYZYmZZZm[Z[erddl\m]Z]m^Z^m_Z_e`eaZbejcdeadZeejcdeadZfejcdeadZgeTjhZiegd ZjejkGd!d"d"ZlGd#d$d$elZmGd%d&d&elZnd9d+d,Zoed-eQeQd.ZpGd/d0d0eSepeepZqGd1d2d2e<Zrejkd3d4Gd5d6d6ZsGd7d8d8etZudS):) annotationsN)Counter)AnyCallableGeneric no_type_checkOptional TYPE_CHECKINGUnion)TypeVar)immutable_dict) OrderedSet)FloorDivIdentityModularIndexing)free_symbol_is_type prefix_strsymbol_is_typeSymT)counters)configir scheduler)prologue_preserves_zero_mask) code_hash) MemoryDepStarDepWeakDep)IRNodeTritonTemplateBuffer)!indexing_dtype_strength_reduction) green_text yellow_text)BaseSchedulerNodeBaseScheduling WhyNoFuse) cache_on_selfexpr_fits_within_32bitget_dtype_sizeIndentedBuffer Placeholderprefix_is_reduction'set_kernel_post_grad_provenance_tracingsympy_index_symbol sympy_product sympy_subsunique)ops OpsWrapperV)BlockPatternMatcher) CSEVariableindex_prevent_reorderingKernel PythonPrinter) MultiKernel)DisableReductionEnableReductionNodeScheduleMarkerSIMDKernelFeatures)IterableIteratorSequenceZ perf_hintsZscheduleZfusion)zyxr0_r1_csheZdZdZejjejjddfddZee e dddZ dddZ ee e d ddZ ZS)!IterationRangesa Each range tree represents multiple sets of iteration indexing in a single tiled dimension in the output kernel. If you have two loops ranges one (4, 3, 2) and another (4, 6), then the range tree will be: 4 (i0) 3 (i1) 6 (i3) 2 (i2) Where i0 is shared between both loops, but then the split into different indexing vars. All loop ranges must iterate over the same number of elements. )divisorlengthnamestrvar_listlist[sympy.Symbol] var_rangesdict[sympy.Symbol, sympy.Expr]numel sympy.Exprprefixkernel SIMDKernelrootIterationRangesRootreturnNonec sDt||_||_||_||_||_||_||_||_ | |_ dSN) super__init__rLrNrPrRrTrJrKrUrW) selfrLrNrPrRrTrUrJrKrW __class__K/var/www/auris/lib/python3.10/site-packages/torch/_inductor/codegen/simd.pyr]`s  zIterationRanges.__init__boolcC t|jSr[)r-rTr^rararb is_reductionxs zIterationRanges.is_reduction sympy.SymbolcCrdr[)r/rLrerararbsymbol~ zIterationRanges.symbolrcCsddtD}||jS)NcSi|]\}}||qSrara).0symtrTrararb z(IterationRanges.symt..)ritemsrT)r^Zprefix_to_symtrararbrls zIterationRanges.symt)rLrMrNrOrPrQrRrSrTrMrUrVrWrXrYrZrYrcrYrg)rYr)__name__ __module__ __qualname____doc__sympySOner]propertyr(rrfrhrl __classcell__rarar_rbrIPs rIcsheZdZ d.d/fdd Zd0ddZd1ddZd2ddZd3d!d"Zd4d&d'Zd5d)d*Z d6d,d-Z Z S)7rXNrLrMrRrSrTindexintrUrV pid_cacheOptional[dict[str, str]]is_looprc tensor_dim Optional[int]grid_dimhas_zdimrYrZc sj|duri}tj|gi||||d||_i|_||_|r'|jr%| dus'J||_||_| |_| |_ dS)N)rLrNrPrRrTrUrW) r\r]r{nodesr}rfrrrr) r^rLrRrTr{rUr}rrrrr_rarbr]s&  zIterationRangesRoot.__init__cCsd|jd|jdS)NzIterationRangesRoot(, z, ...))rLrRrerararb__repr__szIterationRangesRoot.__repr__cCs|jD]}|qdSr[)rvalues cache_clear)r^noderararbrs zIterationRangesRoot.cache_clearrgcCst|jdS)Nr{)r/rTrerararb index_symszIterationRangesRoot.index_symrJrKIterationRangesEntrycCstjj|||jrt||}nt|||}||jvrMt |j t tj j ||||}|tj j|<|j|||j|<||j|<|j|S)zF Lookup a given RangeTreeEntry, creating it if needed )r5graphsizevarsstatically_known_equalsrRrrrrrrTnextrUiter_vars_countrange_tree_nodesrhrNappendrP)r^rJrKexprrrararblookups    zIterationRangesRoot.lookuplengthslist[sympy.Expr]list[IterationRangesEntry]cCs@tjj}g}t|D]}||||||}q gt|Sr[)rvrwrxreversedrr)r^rrJitervarsrKrararbconstruct_entriess    z%IterationRangesRoot.construct_entriesrOcCsdd||DS)NcSg|]}|qSra)rh)rkerararb z1IterationRangesRoot.construct..)rr^rrararb constructzIterationRangesRoot.construct+tuple[list[sympy.Symbol], list[sympy.Expr]]csdd|jD}fdd|D}|jdddtjjggfdd}|D]}tjj|j sE| t |j |j ||q+tjjj s_| t j gt gt fS) z,Figure out vars from this tree used in indexcSsg|] }tjj|qSra)r5rUrgetrksrararbrsz6IterationRangesRoot.vars_and_sizes..cs g|] }|r|jjkr|qSrarTrknrerarbrs cSstjjj|jtjdS)N)fallback)r5rr size_hintrJrZunbacked_symint_fallbackrFrararbsz4IterationRangesRoot.vars_and_sizes..keycs(||j|jdSr[)rrhrK)r)rJ index_varssizesrarbadds z/IterationRangesRoot.vars_and_sizes..add) free_symbolssortrvrwrxr5rrrrJrrrRr)r^r{rrrra)rJrr^rrbvars_and_sizess" z"IterationRangesRoot.vars_and_sizesr[)rLrMrRrSrTrMr{r|rUrVr}r~rrcrrrrrrcrYrZrYrMrYrZrq)rJrSrKrSrYr)rrrYr)rrrYrO)r{rSrYr) rrrsrtr]rrrrrrrrzrarar_rbrXs *     rXcsdeZdZd fd d Zd!d dZd"ddZd#ddZd!ddZd$ddZd%ddZ d&ddZ Z S)'rrLrMrJrSrKrparentrIrYrZc sPtj||j||j|j|j|||j|jd ||_t d|j |_ ||_ dS)N) rLrRrNrPrTrJrKrUrW)r\r]rRrNrPrTrUrWr functools lru_cache_codegencodegenr)r^rLrJrKrrr_rarbr]s  zIterationRangesEntry.__init__c Cs.d|jd|jd|jd|jd|jd S)NzIterationRangesEntry(r))rLrJrKrrPrerararbrs.zIterationRangesEntry.__repr__cs$fdd|_dd|j_|_dS)NcsSr[rararLrarbr z/IterationRangesEntry.set_name..cSdSr[rararararbr!r)rrrL)r^rLrarrbset_names  zIterationRangesEntry.set_namecCs|jdSr[)rrrerararbr$sz IterationRangesEntry.cache_clearcCstj||jSr[)r5rUcodegen_iteration_ranges_entryrLrerararbr's zIterationRangesEntry._codegenrcCsg}t|jtjr |St|jttfsJt|j|jjddD]"}t|tjtjfsD|j }t |dkrDt dd|DrD| |q"|S)Nr6rcss|] }t|tjVqdSr[)rrSIZErrararb 4s  z8IterationRangesEntry.precomputed_args..) isinstancerrvSymbolrrtypeargsIntegerrlenallr)r^precomputed_argsargsymbolsrararbr+s z%IterationRangesEntry.precomputed_argsr|cCrdr[)hashrLrerararb__hash__:rizIterationRangesEntry.__hash__otherobjectrccCst|tsJ|j|jkSr[)rrrL)r^rrararb__eq__=s zIterationRangesEntry.__eq__) rLrMrJrSrKrSrrSrrIrYrZr)rLrMrYrZr)rYrrYr|)rrrYrc) rrrsrtr]rrrrrrrrzrarar_rbrs      rvalueUnion[int, float]rYrMcCs6|tdkrdS|tdkrdSt|rdSt|S)Ninfz float("inf")z-infz float("-inf")z float("nan"))floatmathisnanrepr)rrararb constant_reprBs   rCSEVariableType)bounddefaultcseZdZUdZeZded<ded<dZded<ded < ddfdd Ze e e dddZ dddZ e ddd Zdd!d"Zdd(d)Zdd+d,Zdd/d0Zdd6d7Zdd8d9Zdd:d;Zdd=d>Zdd?d@ZddBdCZddEdFZddGdHZddIdJZddMdNZddOdPZddRdSZddVdWZe dd\d]Z!e"e#j$j%fdd_d`Z&ddbdcZ'e"ddddeZ(ddfdgZ)ddhdiZ*ddjdkZ+ddldmZ,dddodpZ-ddrdsZ.ddtduZ/dddxdyZ0e1j2dd~dZ3dddZ4e ddZ5ddZ6ddZ7ddZ8ddZ9ddZ:ddZ;dddZ<Z=S)rVzo Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests. zCallable[[sympy.Expr], str]sexprkexprFrcallow_block_ptrrM kernel_nameNtilingdict[str, sympy.Expr]featuresr@r}r~override_persistent_reductionOptional[bool]override_cooperative_reductionrYrZcs|duri}t|_|_t_t_dd|D_ g_ i_ t _|_|dur;|n_|durF|n__d_tddfdd }|_|dS)NcSs i|] \}}|tjj|qSra)r5rrsimplify)rkrTvalrararbrmhsz'SIMDKernel.__init__..r{rScs6tjj|}jD]}||}q |Sr[)r5rrsimplify_with_rangesrP range_treescombine_contiguous_dimscombine_modular_indexing_pairs)r{treererarbsimplify_indexing}s  z.SIMDKernel.__init__..simplify_indexing)r{rS)r\r]rZ get_mutationsZ mutationsr+bodyZ indexing_coderonumelsrr itertoolscountrrfinside_reduction should_use_cooperative_reductioncooperative_reductionshould_use_persistent_reductionpersistent_reduction want_no_x_dimno_x_dimrrrrinitialize_range_tree)r^rrr}rrrr_rerbr]Ys8     zSIMDKernel.__init__r|cCtdd|jDS)Ncs|]}t|VqdSr[)r-rkrTrararbrz0SIMDKernel.num_reduction_dims..)sumrrerararbnum_reduction_dimsszSIMDKernel.num_reduction_dimsdtype torch.dtypecCtr[NotImplementedError)r^rrararb dtype_to_strzSIMDKernel.dtype_to_strcCs||jSr[)rrZselect_index_dtypererararb index_dtypeszSIMDKernel.index_dtypecCdSNFrarerararbrr zSIMDKernel.want_no_x_dimrrfrrlist[IterationRangesRoot]cstfddtD}| p| }d dd}gd} dd g} |r#| } n |r(| } n| | } || |} || t} g}t|D]6\}}t|}| |}| |}|durT|n|}|t|d ||||||oi|j ||d vd q<|S)Nc3|] }|vr|VqdSr[rarrrarbrsz3SIMDKernel.construct_range_trees..rYdict[Any, int]cs ddtfdd|DDS)NcSrjrara)rkidxrrararbrms zPSIMDKernel.construct_range_trees..filtered_index_map..c3rr[ra)rkrmaskrarbrzOSIMDKernel.construct_range_trees..filtered_index_map..) enumerate)seqrrarrbfiltered_index_mapsz.filtered_index_map)rFrErDrGrHr{rD)r}rrrr)rYr)r all_prefixesrr-rrrXr)r^r}rrfrrZactive_prefixesZno_r_dimrZ grid_dimsZreduction_dimsZ tensor_dimsZtensor_dim_mapZ grid_dim_maprirTrrr{rarrbconstruct_range_treessF        z SIMDKernel.construct_range_treesdict[str, str]cCs.|||j|j|j|j}|j|dSr[)rrrrfrrrextend)r^r}rrararbrsz SIMDKernel.initialize_range_treeindicesSequence[sympy.Expr]cCr )zr Hook called right before codegen with every index that will be used in the fused kernel. Nra)r^rrararbfinalize_indexingrzSIMDKernel.finalize_indexingrLr{rSrr8cCs,|j}d|_z ||||W||_S||_wr )rstore)r^rLr{rpriorrararbstore_reductions zSIMDKernel.store_reductioncCr r rarerararbrr z+SIMDKernel.should_use_cooperative_reductioncCr r rarerararbrr z*SIMDKernel.should_use_persistent_reductionrQcCsttjdd|jDS)Ncss|]}|jVqdSr[)rProrkrrararbrs  z(SIMDKernel.var_ranges..)dictrchain from_iterablerrerararbrPs  zSIMDKernel.var_rangescCr)Ncss|] }t|jduVqdSr[)r|rr#rararbrsz0SIMDKernel.triton_tensor_ndim..)rrrerararbtriton_tensor_ndimrzSIMDKernel.triton_tensor_ndimrcCs(dg|}d||<dd|dS)NrZ:[r])r'join)r^rrrararbindexing_size_strszSIMDKernel.indexing_size_str list[str]cCsLdg|}|jD]}|jdurq |jr|jr#|jd||j<q |S)N1ZBLOCK)r'rrrfrrTupper)r^rrrararbdense_size_lists   zSIMDKernel.dense_size_listcCs|}dd|dS)Nr)rr*)r0r+r^rrararbdense_size_strszSIMDKernel.dense_size_strcCsxt|ts|S|jd}|j|}dur|St|||ji}tjj |}t||j |j tjj|j jiS)Nr)rrrrrr1rr5rrrrWrrrvrwrxrRrh)r^r{rFZ tree_node new_indexrararbr s   z)SIMDKernel.combine_modular_indexing_pairsrrXcCs8tjj|}r|\}}t||||S|||Sr[)r5rrZexpand_floor_divr_combine_contiguous_dims)r^r{rZ expand_resr3 denominatorrararbrs z"SIMDKernel.combine_contiguous_dimsc Cst|tjtjfr |S||\}}t|dkr|Stjj ||t |g||\}}}||kr1|S| |}t |t t|||} | S)zI More aggressive simplification to merge contiguous dims r6)rrvrrrrr5rrZ_simplify_loopsr9rr1r$zip) r^r{rrrZ new_sizesZreindexZ_pruneZnew_index_varsr3rararbr4$s   z#SIMDKernel._combine_contiguous_dims'contextlib.AbstractContextManager[None]cs,jdjpjtjfdd}|S)Nc3sfjsjr JdVdSrd_zdVr)Wd_dSWd_dSd_w)NFT)rrfr codegen_bodyrar^Z should_flushrarbctx;s    z)SIMDKernel.disable_reduction..ctx)rrr contextlibcontextmanager)r^r;rar:rbdisable_reduction8szSIMDKernel.disable_reductionrrOcGs,t|t|jks Jddt||jDS)NcSsg|] \}}||qSra)r)rkrKrangesrararbrRsz)SIMDKernel.set_ranges..)rrr6rrararb set_rangesPs zSIMDKernel.set_rangesgroupsIterable[sympy.Expr]Sequence[Sequence[sympy.Expr]]Stuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]]c stdd|Drdd|DgfStjjdd|Dfdd|Dtdfd d }ddd}g}d}|D]~}g}|D]r}|drU|ddqE|tkru|dru|d7}|tkru|dsc|dtkr ||r ||st |} t ||} ||| ||| ||d| qE|t |||qE||q?tddDsJdd||fS)Ncss|] }t|dkVqdSrNr)rkrKrararbr^rz5SIMDKernel._split_iteration_ranges..cSg|]}gqSrara)rkgrouprararbr_z6SIMDKernel._split_iteration_ranges..cSrGrara)rk_rararbrbrIcsg|]}|qSrar)rkg)svrarbrcrnrr|rrSrYcsF|}||stt|||<||tSr[)rstatically_known_multiple_of CantSplitrrr)rr new_ranges remainingrMZ var_countrarb add_rangefs z5SIMDKernel._split_iteration_ranges..add_rangesizeidx1idx2(Callable[[list[sympy.Expr]], sympy.Expr]csdfdd }|S)N flat_varsrrYrScs||Sr[ra)rXrUrVrTrarbgetterrrzISIMDKernel._split_iteration_ranges..make_combined..getter)rXrrYrSra)rTrUrVrZrarYrb make_combinedosz9SIMDKernel._split_iteration_ranges..make_combinedrr6cSstjjSr[)rvrwZZero)rJrararbr}sz4SIMDKernel._split_iteration_ranges..css"|] }tjj|dkVqdS)r6Nr5rrrrrararbr zfailed to set ranges  )rr|rrSrYr|)rTrSrUr|rVr|rYrW)rr5rrrrrrrZstatically_known_gtrNrOroperator itemgetter) rArrSr[return_getters_groupsZ current_groupZ length_groupZreturn_gettersrTZsize1Zsize2rarPrb_split_iteration_rangesWsb    z"SIMDKernel._split_iteration_rangesreduction_numelcCsjtjj}t|ddkr!|t|t|d|r!|d|gf}z |||WdSty4YdSw)Nr6rTF)r5rrrrr0rbrO)clsrArrcrrararb is_compatibles  zSIMDKernel.is_compatiblelist[list[sympy.Expr]]cCsPdd|jD}|js|D] }t|rtjj||<q g|}||||jS)NcSsi|]}|j|jqSra)rTrR)rkrtrararbrmrnz3SIMDKernel.split_and_set_ranges..) rrr-rvrwrxrmap_kernel_groups_to_node_sizesr@)r^rrrTrArararbsplit_and_set_rangess  zSIMDKernel.split_and_set_rangescsft|t|krtddt||Dr||S|||\}}gtj||fdd|DS)a We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1). To do this we need to split up the iteration space of i0 into something like: for i1 in s0: for i2 in s1: i0 = i1*s1 + i2 .... This function matches and resplits lengths to the groups of this kernel to enable tiled + non-tiled fusions. css.|]\}}tjjt||dkVqdSrEr5rrrr0)rkrFrLrararbrs  z=SIMDKernel.map_kernel_groups_to_node_sizes..csg|] }fdd|DqS)csg|]}|qSrara)rkfnrrarbrrzISIMDKernel.map_kernel_groups_to_node_sizes...ra)rkfnsrlrarbrsz>SIMDKernel.map_kernel_groups_to_node_sizes..)rrr6rbrr%r&)rdrArr@rQrararlrbrhsz*SIMDKernel.map_kernel_groups_to_node_sizescCs t|tjSr[)rrTMPr^r{rararbis_indirect_indexings zSIMDKernel.is_indirect_indexingcs||rdSdgt|j}|jD] }||jvrq|j|}t|jts'J||jj|j 9<qt j j j tfddt||jDS)NFr6c3s$|] \}}||kVqdSr[ra)rkZ idx_rangeZ iter_rangerKrarbrs  z,SIMDKernel.is_broadcasted..)rprrrrrrrXr{rKr5rrranyr6r)r^r{Z index_numelsrhentryrarKrbis_broadcasteds      zSIMDKernel.is_broadcastedcCs4t|trddt|j|dS|||S)a Convert an index expr to a string that can be used in output code. e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel. Index expressions often need to be passed in as arguments to the triton kernel. Rename_indexing and codegen_indexing keep track of the needed indices and add new parameters to the function signature. r)rr*)rlistr+map index_to_strrZrename_indexingrorararbrvs zSIMDKernel.index_to_strcCs||}t|tjjj}t|tj st|tj r%| tjjj}t|tj rV|tj D]"}|j }t|dkrUt dd|DrU|tjj|i}t||}q3||}t|tsb|n|jd}||S)Nrcss"|] }t|tjtjfVqdSr[)rrrZPRECOMPUTED_SIZErrararbrs  z.SIMDKernel.prepare_indexing..)rr1r5rrZprecomputed_replacementsrZatomsrvfloorZceilingsubsrrlookup_precomputed_sizerrrcodegen_indexing)r^r{ar replacementsZ simp_indexrararbprepare_indexing s"     zSIMDKernel.prepare_indexingreordercsfddjD}|rIt|dkrItdd|D}ddd|d|Dd| dks=Jd d|d|Dt|d||d|<|S) Ncsg|] }|jr jr|qSra)rfrrktrerarbr2sz1SIMDKernel.active_range_trees..r6css|]}|jdvVqdS)ZxyzNrrrararbr6sz0SIMDKernel.active_range_trees..css|]}|jVqdSr[rrrararbr7sZzyxcSg|]}|jqSrarrrararbr7s)rrrr+r)r^r~Ztreesrrarerbactive_range_trees1s 2 zSIMDKernel.active_range_treesrcCstjj||}t|jtdD]6}||jvrGi}|j| D] }tjj |||<q!t |dkr@t |j|j ||j|_ |j|q|S)Nrr)r5rrrrPsortedrrMrrryrr1rr)r^rsymr|Zpsrararbrz=s    zSIMDKernel.codegen_indexingcCtd)NzNYI: codegen_nan_checkrrerararbcodegen_nan_checkNzSIMDKernel.codegen_nan_checkrOptional[IRNode]cCr)NzNYI: call_kernelr)r^rLrrararb call_kernelQrzSIMDKernel.call_kernelrUnion[str, OpsWrapper]r Iterator[str]ccs\|j}|j}|rt||}t|}||_||_z |VW||_||_dS||_||_w)z:Context manager to add an additional mask to tl.load/storeN)Z _load_maskZ _load_otherr3 logical_andr4_unwrap)r^rrr!Z prior_valrararb mask_loadsTs   zSIMDKernel.mask_loadscCs\dd|jD}t||}i}|jD]}t|j}t||dit||di||<q|S)a\ This gets the stride of the index for each of the tiling variables (technically, it does it at index 0) For example, if xindex = x0 + 512*x1 + 1024*r0 x0 = (xindex//512) x1 = (xindex % 512) r0 = rindex // 1024 this function would return {xindex: 512, rindex: 1024} cSsi|]\}}||jqSra)r)rkkvrararbrmvz2SIMDKernel.get_strides_of_load..r6r)rror1rr/rL)r^r{Zindex_to_tile_indexesZindex_in_tile_varsstridesZ range_treerrararbget_strides_of_loadhs    zSIMDKernel.get_strides_of_loadcCs t|tr tt||S||Sr[)rtupleru)rkrrararb_map_tuple_or_scalars zSIMDKernel._map_tuple_or_scalarcCs0g}tt|jj}|j\}}}}|j}tj j t |j }t|D]i\}}||vr8|dq*tj |} tj j | } | |krxtt} d} ||D]} t| ttfrj| d| | d7} qT| | jqTt| |}n| }tj |}t|}|||dt||kq*t|S)a+ Try the best to estimate the total size (in bytes) of the kernel's inputs and outputs, which is used for estimating the memory throughput of this kernel. This information is used for checking how far we are from the peak memory bandwidth. It's important that we want to avoid overestimating the sizes of the inputs and outputs, because it can wrongfully give us a very large memory traffic value, which may be even larger than the theoretical bandwidth and thus become very misleading. This is particularly problematic for cases where we slice some inputs. In those cases, we should only count the size of the "slices" instead of the original inputs, because only the slices contribute to the real memory traffic. rZ no_index_dep_r6)rr2rinplace_buffersrpython_argdefsr buf_accessesr5rrrr0rrrZ get_numelr rrrrrr{Z get_dtyper*r|r)r^nbytesZ ninplace_argsrJ call_argsrZ out_numelrrZ arg_numelZbuf_sizerZno_index_dep_countdeprRrZ dtype_sizerararbestimate_kernel_num_bytess2        z$SIMDKernel.estimate_kernel_num_bytescCstt|jjdkrt|jjdkrt|jjdkrdS|j\}}}}d}|D]}tj|}|s2q'| } t| j dkrtdd| j DdkrJq't | j } |durW| }q'|| krtd|d d | d |} t| d d|D} d d|D} dd|D}dd|D}td|d|d| d| d|d} t| dSq'td|d} t| dS)zr Print message if the kernel have mixed layout inputs. Only care about 4D tensor for now. r6rNcSsg|]}|dkr|qS)r6rarkrFrararbrrz.SIMDKernel.warn_mix_layout..rzExpected stride order z, but found stride orderr^z for kernel cSs4g|]}tj|rttj|jndqSr[)r5rtry_get_bufferrget_stride_order get_buffer get_layoutstriderkrLrararbrs cSs.g|]}tj|rtj|jndqSr[)r5rrrrrTrrararbrs  cSs0g|]}|tjjvr dn |tjjvrdndqS)Z GraphInputZIntermediateBufferN)r5rZ graph_inputsZname_to_bufferrrararbrs  cSrrarrrararbrz param names z buf names z strides z sizes z sources  z%All the inputs for the triton kernel z have uniform layout)rrZ input_buffersZoutput_buffersrrr5rrrrTrrrr$logwarningr#)r^rZargdefsr _signaturerJZuniform_stride_orderZarg_namebufZlayoutZ stride_ordermsgZstride_order_listZ size_listZ source_listZ argdef_namesrararbwarn_mix_layouts\       zSIMDKernel.warn_mix_layoutc Cspt||d|}d|_t|jj|}t||}d|_t||}t||}t||d|}t |||fS)NrFT) r3 reductionrZ index_exprrrctruedivsubmulr4r) r^rrZsum_rnumelmeanZdxZdx2m2rararbwelford_reduce_fallback s   z"SIMDKernel.welford_reduce_fallbackcCsDt||d|}t||}t|}t||d|}t||fS)Nmaxr)r3rrexpr4r)r^rrZvmaxrrZvsumrararb prepare_softmax_twopass_fallbacks   z+SIMDKernel.prepare_softmax_twopass_fallbackcCrr[rrerararbcodegen_kernelr zSIMDKernel.codegen_kernelcCrr[rarerararbr9"r zSIMDKernel.codegen_bodyrrrcCrr[ra)r^rrrararbr%r z)SIMDKernel.codegen_iteration_ranges_entry)NNN) rrrr@r}r~rrrrrYrZr)rrrYrMrrp) r}r~rrcrfrcrrrrcrYr )r}rrYrZ)rrrYrZ)rLrMr{rSrr8rYrZ)rYrQ)rr|rYrM)rYr-)r{rSrYrS)r{rSrrXrYrS)rYr7)rrSrYrO)rArBrrCrYrD)rArBrrCrcrSrYrc)rrCrYrf)rArrrCrYrf)r{rSrYrc)r{rSrYrMF)r~rcrYr )rrSrYrSrr[)rLrMrrrYrZ)rrrrrYr)r{rSrYrQ)rrr)>rrrsrtrupexprr__annotations__rr]ryr(rrrr rrrrr"rrrPr'r,r0r2rrr4r>r@ staticmethodrb classmethodrvrwrxrerirhrprsrvr}rrzrrr<r=rrrrrrrrr9rrzrarar_rbrVOs|   /     6            L      &     ?G rVc@s.eZdZUeZded<ddZddZeZeZ ddZ dQd d Z e dRddZ dSddZdTddZddZdddUd!d"Zd#d$Z dVdWd,d-Zd.d/Zeed0dXd2d3ZedYd8d9ZedZd.)rr1rararbgroup_fn,zSIMDScheduling.group_fncsdt|tjs t|tjrtj||S|j\}\}}|j\}\t||}|r7|s7|r6|dn|rG|sG|rG|d|rc|rc|koV|k}|sa|d|||S|s7|s7|kru|ks|s|d||dS| D]+}|rn$| | @sq|j\}\} } || kr|| ks|d|| || dSqt ||fdD]\} } | rt| t} | s|| d| Sq|| ||}|| ||}|| | ||}tjjr5d}t|d krt|d kr||ko|kn}n||k}n t|d kr)||k}|s5|d |||dSdS|s|r|d krKd ksMJ|krtfd d | Dsi|ddStjjr|st|| ||d fd ffv}|s|d|SdS|kr|d|kS|r|rJ|||S)z Hook called by Scheduler to determine if the Triton backend can fuse node1 and node2. These nodes might already be FusedSchedulerNodes. z&Split scan cannot fuse with reductionsz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)z5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)Fz:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s))node1node2z is not TritonTemplateBufferTrztiling mismatch (%s, %s, %s)r6c3s$|] }tf|VqdSr[)rVre get_rangesrZnumel2Zrnumel2rarbrs  z*SIMDScheduling.can_fuse..z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)rrZForeachKernelSchedulerNodecan_fuserHr'Z is_split_scanrf is_template get_nodesused_buffer_namesget_buffer_namesr6Zget_template_noder! select_tilingrtritonZ tiling_prevents_pointwise_fusionrrZ tiling_prevents_reduction_fusionrrcan_fuse_horizontal)r^rrrJZnumel1Zrnumel1whyZreduction_can_fuserZ pro_numelZ pro_rnumelrZ node_nameZis_triton_templatetiling1Ztiling2Ztiling3ZcondZis_reduction_tiling_validrarrbr/s         zSIMDScheduling.can_fusec sbgttjttttdfdd}fdd}fdd}fdd}tjfd d }fd d } |D]h} | vrMqF| || r| | rn|Wdn1siwYr{|| s{pytnd|| qF|| r| | Wdn1swYqFt d dd| j dS)Ncs2|j\}\}}|kr|kp|ko|dkSNr6rHrrJZ node_numelZ node_rnumelrRrrarbfits_in_main_bodysz@SIMDScheduling.generate_node_schedule..fits_in_main_bodycs&|j\}\}}|ko|dkodkSrrrrrarbfits_outside_reductionszESIMDScheduling.generate_node_schedule..fits_outside_reductioncs"|jjD] }|jvrdSqdS)NTF) read_writesreadsrL)rread)current_loop_buffer_usagerarbexpect_improved_memory_usages  zKSIMDScheduling.generate_node_schedule..expect_improved_memory_usagecs||dd|jjD|r8t|tjr8t|j t j r8t|j j t j s8|dSdd|jjDdS)NcSrrarrrararbrrzXSIMDScheduling.generate_node_schedule..schedule_node_in_loop..cSrrarrrararbrr)rrupdaterrrfrr SchedulerNoderrZComputedBufferdataZScanget_namewrites)r)rdone node_schedulenot_ready_yet_nodesrarbschedule_node_in_loops    zDSIMDScheduling.generate_node_schedule..schedule_node_in_loopc3snrdturntr%tdtddVtdS)Nr8r6)r>poprr=insertclearra)rmaybe_split_indexrrrarbend_current_reduction_loops     zISIMDScheduling.generate_node_schedule..end_current_reduction_loopcs<dkrdS|j@s dS|rt|dttfrJtS)Nr6Fr8)Z ancestorsrr>r=rc)rr)rrrarb#requires_closing_previous_reductions  zRSIMDScheduling.generate_node_schedule..requires_closing_previous_reductionzunexpected group: (rz) != r6) r rr%rMr<r=rrrrrH) r^rrRrrrrrrrrra)rrrrrrRrrbgenerate_node_schedulesD          z%SIMDScheduling.generate_node_scheduler.rz Schedule: %s)rrrHr schedule_logdebugcodegen_node_scheduler@)r^rrrJrRrrrararb codegen_nodes  zSIMDScheduling.codegen_noderRrSbuffers.csrr[)r))rkrTrararbr>rz8SIMDScheduling.can_use_32bit_indexing..T) torchZiinfoZint32rr)rr5rrZ guard_leq)rRrZint_maxZ buf_sizesrTrararbcan_use_32bit_indexing,sz%SIMDScheduling.can_use_32bit_indexingkernel_featuresr@c Cs"|j}|||j|j}|||gd|i}|D]}|||qt||D]9}t | | }Wdn1s>wY| |||}t j jrSt||td|||_t||_q(~t|dkrnt|}n|\}t ||D]} | q{Wdn1swY||||jt jr|t jr||djtjj|jO_tjj|jO_tjjj r t j!r |dj"#} |D]5} | $} | | vrq| j%dusJ| j%&} | dur t'ddd7<tjj(d| j)d| d q|*dS) Nrz+Generating kernel code with kernel_name: %sr6rZinductorZintermediate_hookszrun_intermediate_hooks(rr)+rrrRrccreate_kernel_choices!codegen_node_schedule_with_kernelr<Zmerge_workspaces_inplacer5set_kernel_handlerr define_kernelrtraceenabledr.rrrrrZscheduler_nodesmark_runcodegen_commentrZ nan_assertsrrrremoved_buffersinplaced_to_remove wrapper_codeZsupports_intermediate_hooksZgenerate_intermediate_hooksrZlive_output_buffersrrZget_origin_noder writelinerLfree_buffers_in_scheduler) r^rrrZkernelsrUsrc_coderZ final_kernelrZ live_outsrLZ origin_noderararbrHst                  z$SIMDScheduling.codegen_node_schedulelist[SIMDKernel]cCs|j|i|gSr[)r)r^rZ kernel_argsZ kernel_kwargsrararbrs z$SIMDScheduling.create_kernel_choicesc Cs|tt}i}|D]0}|tur||q |tur"|q ||| }| t |j |q |||D](}|turS||qE|tur\|qEt|j || }||qEWddS1sywYdSr[)r< ExitStackr= enter_contextr>r>closeZdecide_inplace_updaterirrr$fromkeys_bodyZindexing_from_argsrrkeysr"r)r^rrUstackZ all_indexingrrrararbrs4    "z0SIMDScheduling.codegen_node_schedule_with_kernelFonly_gen_src_code Optional[str]c Cs|j\}\}}|dks J|j|j\}} i} |} g} |D]+} | }| | || @rKt|dks7J| | tt|<|j tt|g} q t| dksTJ||se|g|D]}| q^| }| d|D] }| ||qp|jtWdn1swY|jD]|\}}d|d}| |g} rtdd| D}td | N| |9| D]'}t|dkrt| dkrt|r|j|O_| ||q|jtWdn1swYWdn 1swYqWdn 1s wYt|ts7|d |jd d d t ||j!D]}d|d}|j|d d qB| dt|trb|}n|d|j"}Wdn 1suwYg|||}tj#r|$d}|%d|d|&|'}|r|WdS|(|||}tj)j*rt+||Wdn 1swY|,||-||jtj.j/|j/O_/tj.j0|j0O_0|1dS)z Codegen a triton template If `only_gen_src_code` the src code will be returned instead of codegen'd into the wrapper r6rzNz cs|]}|VqdSr[)Zcan_codegen_without_upcasts)rkZp_nrararbrs z2SIMDScheduling.codegen_template..ztriton.codegen_upcast_to_fp32z z F)strictgeAr)2rHrZmake_kernel_renderrrrrriterZprologue_fused_inputsrrZset_subgraph_bodyrrirZcseZ invalidater Znamed_input_nodesrorrrrpatchrZ#prologue_fused_inputs_preserve_zerorrMZ finalize_hookr5rr codebenchmark_kernelrZimports_for_benchmark_kernelZcodegen_kernel_benchmarkgetvaluerrrr.rrrrrr)r^Z template_nodeZepilogue_nodesZprologue_nodesrrJZ_numelrrUrenderZbuf_name_to_prologue_groupZtemplate_readsZprologue_groupprologuenamesrZ partial_codeZ input_namebufferZ subgraph_nameZcan_codegen_without_upcastZ prologue_noderrZnum_gbrrararbcodegen_templates         .            zSIMDScheduling.codegen_templatecCstjjtjjdSr[)r5rrrZ device_opsZ synchronizererararb codegen_sync-szSIMDScheduling.codegen_syncsubkernel_nodeslist[BaseSchedulerNode]custom_part_algorithmenable_autotune mixed_sizesrlist[tuple[str, Any, Any]]c Csddlm}dd|D}ii}} t||D]6\} } t| dddj\} \} }|| | |}||| |}||| |f| | <|j|t|| || d|| <q|j ||||| d }t d t |d d|Dg}|D]s}d d|D}|||d }t||D]R\} } | | | d||| || }| | d}|st|t|D]}|qWdn1swYtjj|jO_tjj|jO_q~|}||||fqj|S)Nr6) ComboKernelcSrrarrkrrararbr:rz=SIMDScheduling.generate_combo_kernel_code..cSrr[rrrararbr=rz;SIMDScheduling.generate_combo_kernel_code..r)rZ optimize_mask)rZtriton_schedulingZcustom_algorithmZ kernel_mapZ node_info_mapz1ComboKernels: %d nodes partitioned into %s groupscSsg|]}t|qSrarF)rkprararbrQrcSrrar'r(rararbrUr)r#r$r)Ztriton_combo_kernelr&r6rrHrrZcreate_triton_kernelr@Zhorizontal_partitionrrrrZcreate_sub_kernelr5rr?Z only_nodesrrrrrr)r^r r"r#r$rr&Zfused_node_listsZ subkernel_mapZnode_schedule_mapZpnrrJrRrrrZ partitionskernel_code_listZ node_grouprUZ subkernelrrrararbgenerate_combo_kernel_code0sd          z)SIMDScheduling.generate_combo_kernel_codec Cs|}|j}|j}tjdkptjdko|}|||||}|D]!\}}} |||g|} ||gt d| | t j j | q |dS)Nr6z"ComboKernels: generated kernel %s.)Zget_subkernel_nodesZuse_custom_partition_algor#rZcombo_kernel_allow_mixed_sizesr+rrrrrr5rrr) r^Zcombo_kernel_noder r"r#r$r*rrUrJrrararbcodegen_combo_kernelms     z#SIMDScheduling.codegen_combo_kernel list[CandidateTiling]c sdk}d fdd }|\}t|dkr!tdkr!gS|\}|||r-|n||}fdd |D}|S) Nr6 is_pointwisercrYr.c s t|jt|ksJd|jd||j|jg}tddtj|Ds)Jddtj|D}tdd|jD}dd d }t ||g|dddg}|D]}t j j |j|j} t| t|ksjJz | dd} | t|krzWqTtdd| | dDrWqTWn tyYqTw||d| ||| df} t j j tddt|| D} |j|vr| d9} t | dr| d9} t | dr| d9} t j j | tt|dkr|t ||d| ||| dg| |jdqT|S)zX Compute tiling candidates by dividing up the iteration ranges. zrw.range_vars=z ranges=css|] }t|ttfVqdSr[)rrrrkrrararbrs  zHSIMDScheduling.candidate_tilings..tile_ranges..cSs(g|]}|jtjjvrt|tr|qSra)rLr5rrrrr0rararbrszISIMDScheduling.candidate_tilings..tile_ranges..cSrrarr0rararbrrr?rrYrScSstjjt|Sr[rj)r?rararbcollapse_rangesrzNSIMDScheduling.candidate_tilings..tile_ranges..collapse_rangesnoner)rrLscorer6css|]}|dkVqdSrErarrararbrrNcss |] \}}|dkr|VqdSrEra)rkrTrrararbrsrrr3rL)r?rrYrS)rZ range_varsrrrrr%r&r CandidateTilingcreate_partial_tilingr5rrZ stride_hintsr{ ValueErrorrr0r6rL is_good_sizer) r/r?rwZ dep_sourcesdepsZ write_namesr1tilingsrrsplitZ tiled_groupsr3)rdrcreduction_rangesrarb tile_rangess(          z5SIMDScheduling.candidate_tilings..tile_rangescs*g|]}t|j|j|jdqS)r4)r5complete_partial_tilingrr3rL)rkr)rdrRrcrarbrsz4SIMDScheduling.candidate_tilings..)r/rcrYr.)rrZ"pointwise_or_reduction_read_writes) rdrrRrcr/r>Zpointwise_rangesZpartial_tilingsZ full_tilingsra)rdrRrcr=rbcandidate_tilingss ^   z SIMDScheduling.candidate_tilings pw_tilingrreduction_tilingrcCsFgdt| d}ddgdt|}tgt||t||S)zK Create a tiling dict from pointwise and reduction splits. )rDrErFNrGrH)rr r6)rdrArBZ pw_prefixesZreduction_prefixesrararb create_tilings zSIMDScheduling.create_tilingrr/cCs||r|ng|s |SgSr[)rC)rdrr/rararbr6 s  z$SIMDScheduling.create_partial_tilingrcc CsHt|}d|v}||}|t|g}|r||fn||f}|j|S)zb Given a tiling for only pointwise or reduction dimensions, adds the missing one. rF)rtrr0rC) rdrrRrcZsplitsr/Z total_numelZmissing_tilingZ tiling_argsrararbr?s  z&SIMDScheduling.complete_partial_tiling"list[dict[str, tuple[sympy.Expr]]]c Cs|dk}ttttjf}t|D]}t|tj sq| }|s+t |ddkr+q||r0dnd}|g} dd|j D} | D]~} g| j} tjj} tjj}t| D]\}\}}| |9} || |rinqW|| |sqqB|d}|r}| d|n| |d}g}|D]3\}}t| j|}td|t|tt |}t||||}|dur|dn|g}| |q| !|qB| D]2}tdt |t"j#j$}|d}t%|d|}|ft&||d}|'|(|)||||qqt*|t dd}|S) z Creates N-dimensional tiling candidiates, attempting to simplify loads/stores by tiling the kernel into higher dimensions. Returns a list of tilings ranked by dimensionality. r6rcSs(g|]}t|trt|jdkr|qS)r)rrrr?r0rararbrKsz1SIMDScheduling.get_nd_tilings..NrT)rreverse)+r r$rMrvExprr>filterrrrrrrZreads_and_writesr?rorwrxr5rrrZstatically_known_geqrr7Zget_subexpr_involving_symbolr{rrrrZmatch_mod_div_block_exprrrrr max_tilesr0rrr?r6r)rdrZpointwise_numelrcr/r;rZ node_rangesZranges_to_tileZ node_tilingsZ memory_depsrZall_var_rangesZpointwise_vars_numelrZpointwise_end_idxvarrRZreduction_start_idxrPZ index_tilingr{Znum_dimsZ match_resultdimsZ node_tilingZnum_leading_dimsZfirst_trailing_dimZcollapsed_leading_dimZcollapsed_splitsranked_tilingsrararbget_nd_tilings+s        zSIMDScheduling.get_nd_tilingscsdk}||gg}|stjjrtjjdkrBtjtjkr@t |D]}tjjs?t | ||dkr?t td|Sq#|Stt}t}t |D]&}| ||D]} | j|vraqY| jdurl|| j|| | j7<qYqPdd|D} tjjdkr|rdd d} tdt | D]} | | d| | } | dur| g| } nqt | dkrt d| tjjr|||| } | D]ttsJtfdd|Dr߈Sq|S)z Heuristics to decide how to tile kernels. Currently, we tile based on stride-1 dimensions. Returns: `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel` r6rz Reduction over non-contiguous dims. Consider setting config.triton.tile_reductions to True. NcSsg|]\}}|jqSra)r)rkcandidate_tilingr3rararbrsz0SIMDScheduling.select_tiling..rtiling0rrrYOptional[dict[str, sympy.Expr]]cSs|d|dd}}|d|dd}}tjj||dkr#dStjj||dkr;||f||f\}}\}}tjj||dksHJtjj||sRdS|t||||dd}|S)NrFrEr6rrG)rDrErFrG)rr5rrrrNr)rNrZa0Za1Zb0b1Z new_tilingrararbconvert_tiling_to_3dsz:SIMDScheduling.select_tiling..convert_tiling_to_3dzpossibly bad tiling: %sc3s4|]}t|tjrtj|dVqdS))rcN)rrrrVrerrr(rcrrarbrs  z/SIMDScheduling.select_tiling..)rNrrrrYrO)rCrrZtile_reductionsrH perf_hint_loglevelloggingWARNINGr>rGrr@infotextwrapdedentr rM collectionsrrLrr3 most_commonrangeZprefer_nd_tilingrLrr$r)rdrrRrcr/Zdefault_tilingrZ seen_namesZcandidate_tilesrMrKrQrZ new_3d_tilingrarRrbrsv              zSIMDScheduling.select_tilingcCrr[rarerararbflushr zSIMDScheduling.flushcCr r rarerararbready_to_flushr zSIMDScheduling.ready_to_flushc Cs2tdd|Dset|dddj\}\}}||||}||||}|j|t|||d}|||t d|!t | | } Wdn1sPwYWdn1s_wYn)|d |\} } } t d||j| | | d d } Wdn1swY| ttjd } | S) Ncsrr[)rrrararbrrzASIMDScheduling.generate_kernel_code_from_nodes..cSrr[rrrararbrrz@SIMDScheduling.generate_kernel_code_from_nodes..r)rrrTrZtriton_)rqrrHrrrr@rrrr5rrZget_prologue_template_epiloguerreplacerMr,Z KERNEL_NAME) r^rrrJrRrrrrUrrtemplateepiloguerararbgenerate_kernel_code_from_nodess>     z.SIMDScheduling.generate_kernel_code_from_nodescCrr[ra)r^rrararbr4r zSIMDScheduling.codegen_commentcCrr[r)r^rrrUrararbr7r zSIMDScheduling.define_kernelN)rr)rRrSrrrYrc)rr@)rr@rYr)rYrr) r r!r"rcr#rcr$rcrrcrYr%)rYr.)rArrBrrYr)rrr/rcrYr)rrrRrSrcrSrYr)rYrD)rYrrp)&rrrsrtrVrrrrZcan_fuse_verticalrrrrrrrrrrr+r,rrrr@rCr6r?rLrvrwrxrr]r^rbrrrarararbr)sN   `   C #v ={    qr   rT)frozenc@s6eZdZUded<ded<dZded<edd ZdS) r5rrr|r3NrrLcCs"tjj|}|dko|ddkS)z@Somewhat arbitrary heuristic used to boost scores for some sizesr-rr\)rrararbr8AszCandidateTiling.is_good_size)rrrsrtrrLrr8rarararbr5;s  r5c@s eZdZdS)rON)rrrsrtrarararbrOHsrO)rrrYrM)v __future__rrZr< dataclassesrrrUrr_rXrtypingrrrrrr r Ztyping_extensionsr rvrZtorch._loggingZtorch.fx.immutable_collectionsr Ztorch.utils._ordered_setr Ztorch.utils._sympy.functionsrrrZtorch.utils._sympy.symbolrrrrZ _dynamo.utilsrrrrrZanalyze_preserves_zero_maskrZ codecacher dependenciesrrrr r!Zoptimize_indexingr"Zruntime.runtime_utilsr#r$r%r&r'utilsr(r)r*r+r,r-r.r/r0r1r2Z virtualizedr3r4r5Zblock_analysisr7commonr8r9r:r;Z multi_kernelr<Zsimd_kernel_featuresr=r>r?r@collections.abcrArBrC getLoggerrrrZ_loggingZgetArtifactLoggerrSrZ fusion_logZdoprintrr dataclassrIrXrrrrVrr5 ExceptionrOrarararbs  $       4    8{ > _