o ZhD @sddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z ddlZddlmZddlmZddlmmZddlmmZddlmZmZddlmZddlm Z m!Z!gd Z"d e#d e$e#e#ffd d Z%dee&dej'de(e#e ffddZ)dej'de(e#e fdejj*fddZ+d8dejj*d ejj*fddZ,dej*d ej*fddZ-dej*de.ej'de.ej'de.ej'fd d!Z/ej0ej1ej2ej3ej4ej5ej6ej7ej8ej9ej7ej:ej;g ZgZ?ej0ej@ej1ejAej2d"d#iZBde.ej'de(e#ej*ffd$d%ZCde.ej'de(e#ej*fd&e(ej*ej*ffd'd(ZDGd)d*d*ZEd9d-d.ZFd/eEd eGfd0d1ZHGd2d3d3ZIdejJfdejj*d4e e(e#e fd5e&ejJd ejj*fd6d7ZKdS):N) defaultdict)Iterable)Enum)AnycastOptional)ArgumentTarget) ShapeProp)fuse_conv_bn_evalfuse_linear_bn_eval) matches_module_patternreplace_node_modulefuseremove_dropoutextract_subgraphmodules_to_mkldnn reset_modules MklSubgraphgen_mkl_autotuneruse_mkl_length UnionFindoptimize_for_inferencetargetreturncCs*|dd^}}|r|d|fSd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamer!Q/var/www/auris/lib/python3.10/site-packages/torch/fx/experimental/optimization.py _parent_name%sr#patternnodemodulescCst|jdkr dS|jd|f}t||D]2\}}t|tjs"dS|jdkr*dSt|jts3dS|j|vr;dSt ||j|urGdSqdS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)r$r%r&nodes expected_typeZ current_noder!r!r"r /s     r new_modulecCs<t|jtsJt|j\}}|||j<t||||dSN)r+rr/r#setattr)r%r&r3 parent_namer r!r!r"rCs rFmodelc Cs4tjtjftjtjftjtjftjtjfg}|st |}|r&t |t j j s,t |}n|}t|}t |j}|D]W}|jD]Q}t|||rt|jdjdkrTqA||jdj} ||j} | jseqA|dtjtjtjfvrwt| | } nt| | } t|jd|| ||jd||qAqeZdZdedeedfdeeefdeffdd Z Z S)z&remove_dropout..DropoutRemoverrr).kwargsrcs:t|j|tjrt|dksJ|dSt|||S)Nrr)r+Z submodulesr8ZDropoutr(superr')selfrr)rJ __class__r!r"r'{sz2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__r tuplerrAr/rr' __classcell__r!r!rMr"DropoutRemoverzs  rT)r,r@r>Z TransformerZ transform)r7rHrTr!r!r"rts  r orig_moduler1inputsoutputscs|t}i|D] }||j}||<q|D]}||fdd}||<q|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|Sr4r!)xenvr!r"z"extract_subgraph..csg|]}|qSr!r!).0outputrYr!r" z$extract_subgraph..)r,Graph placeholderr Z node_copyr^lintr?)rUr1rVrWrIinputnew_noder%r!rYr"rs     rcCs t|Sr4) th_mkldnnZMkldnnBatchNorm)a_r!r!r"r[s r[cCsi}|D]9}|jdkr=t|jtsJ||j}t|tvr=tt||tj}t|tj s0Jt |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r') r.r+rr/r0 mkldnn_mapr>floatr8Moduler<r=r)r1r& old_modulesr% cur_moduler3r!r!r"rs    rrlcCsJ|D] }|jdkr"t|jtsJ||j}||vr"t||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r'N)r.r+rr/r)r1r&rlr%rmr!r!r"rs  rc@seZdZdejfddZdS)rfx_graphcCs||_g|_g|_g|_dSr4)rnr1 start_nodes end_nodes)rLrnr!r!r"__init__s zMklSubgraph.__init__N)rOrPrQr,rarqr!r!r!r"rsr rcs*dddtdtffdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrCrcs|j}dur|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr!)r>Zrandnshaper]r%r!r!r"r_sz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r)rtr!r!r"r_scs<tD]}|qt}tD]}|qt|Sr4)rangetime)frhbegin)iterswarmupr!r" benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSg|]}|qSr!)to_denser]ir!r!r"r_szRgen_mkl_autotuner..use_mkl_heuristic....cSr|r!) to_mkldnnr~r!r!r"r_r`r!r!Z sample_inputs submoduler!r"r[sz>gen_mkl_autotuner..use_mkl_heuristic..csSr4r!r!rr!r"r[ r\)rornZ owning_modulerlr propagaterlistr,r-rprr1rrCrArB)rCZ input_nodesZ output_argsr{Zmkl_timeZ no_mkl_timeexample_inputsrHryrlrzrr"use_mkl_heuristics" z,gen_mkl_autotuner..use_mkl_heuristic)rbool)rryrzrr!rr"rs rrCcCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r(r1)rCr!r!r"rsrc@sFeZdZddZdefddZdedefddZd ed efd d Zd S)rcCsdg||_dg||_dS)Nrrsize)rLnr!r!r"rqs zUnionFind.__init__vcCs||j|<d|j|<dS)Nrr)rLrr!r!r"make_set s zUnionFind.make_setrcCsB|j|}||kr |S|dusJ|||j|<tt|j|Sr4)rfindrint)rLrparr!r!r"r$s  zUnionFind.findrgbcCsf||||}}||kr|S|j||j|kr ||}}||j|<|j||j|7<dSr4)rrr)rLrgrr!r!r"join,s  zUnionFind.joinN)rOrPrQrqrrrrr!r!r!r"rs r pass_configtracerc s@dddtid}|duri}|||drt|}|dr#t|}|ddur+|St|dts6td d|dvr@td |dd}|}|t |t |j t| }Gd d d t}tjD]}|j} |jd kr||j} t| tvr|j} t| d} | dur| jtjksJd| jtdksJdn|jdkr|jtvr|j} n|jtvr|j} | |jkr"| |jkrtdd|j Dsqk!|t "|j fdd} Wdn1swYt#t$t j%j&| |_ '|(dd|f} |)| |f| _ Wdn 1swYqkt*tj|}|_+jD]B}|jdkrr|jdkrr|j d}t|j,}|D]}|jdkrc|jdkrc|)|-|qKt.|j,dkrr-|q1t.j}t/|fddt0jD]w\}}|jdkr|jdkr||_12|q|jdkr|jdkrĈ|j ddusJ|j d|_3qfdd|j4D}t.|dkrאqtdd|DrJt5|}|d|_6|ddD] }7|d|qqt8fd d}jD]9}t9|d!r|:|j6j;|t9|d"r1|:|j1j<;|t9|d#rC|:|j3j=;|q |>D](}||sp|j<|j=D]}|j d}|)|-|qVt?|j||qId}jD]}|jdks|jdkr|d7}qwt@AtBCd$|Dt |}|S)%a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fusermkldnn_layout_optimizeNrrrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportrrN)rOrPrQNOYESUNKNOWNr!r!r!r" MklSupportbsrr'z)this pass is only for torch.float modulescpuz!this pass is only for CPU modulesZ call_functioncss|]}|jdkVqdS)r}N)r)r]argr!r!r" sz)optimize_for_inference..csd|fS)Nr) call_methodrrnr!r"r[sz(optimize_for_inference..rr}rrcs0t|dr |jSt|dr|jSdS)Ncolor start_color)hasattrrrrr)ufr!r" get_colors    z)optimize_for_inference..get_colorcs,g|]}t|tjr|dur|qSr4)r+r,r-r~)rr!r"r_s  z*optimize_for_inference..css|]}|duVqdSr4r!r~r!r!r"rsrcstSr4)rr!rr!r"r[r\rr end_colorzmkldnn conversions: %s)Erupdaterrr+rA RuntimeErrortracer<r=r,r?rootrBrrr1rr.rr0mkldnn_supportedrnext parametersZdtyper>rjZdevicemkldnn_supported_unknownranyr)Zinserting_beforeZmap_argrrRr%rZinserting_afterZ create_noderErrlrDrFr(r enumeraterrrZall_input_nodessortedrrrrrappendrorpvaluesrlogging getLoggerrOinforc)r7rrZdefault_pass_configrZ cur_tracerr&rr%Zsupports_mkldnnrmZsample_parameterZ mkldnn_argsZdense_xrlZprv_noderDuserZ num_nodesZcur_idxZ cur_colorsZ other_colorZ mkldnn_graphsrCZprvZmkldnn_conversionsresultr!)rnrrr"r6s                                      r)FF)rrr)Lr<roperatorrv collectionsrcollections.abcrenumrtypingrrrr>Ztorch.fxr,Ztorch.nnr8Ztorch.nn.functionalZ functionalFZtorch.utils.mkldnnutilsZmkldnnrfZ torch.fx.noderr Ztorch.fx.passes.shape_propr Ztorch.nn.utils.fusionr r __all__r/rRr#r0r-rAr rkrrrrrr9r;r:ZReLUZ MaxPool2dZ AvgPool2dZAdaptiveAvgPool2dZreluZ transposeZsigmoidZ avg_pool2dZadaptive_avg_pool2draddmulrZ MkldnnConv2dZ MkldnnLinearrirrrrrrrZTracerrr!r!r!r"s            (   "   .