o Zh6@s>ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl Z ddlmZddlmZddlmmZddl mZddlmZddlmZdd lmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$e%e&Z'd d Z(eddZ)edej*de fddZ+d1ddZ,ddZ-edej*de fddZ.Gdddej/Z0edej*de fddZ1edd Z2d!d"Z3e j4j5Z5e5j6e5j7e5j8e5j9e5j:e5j;e5je5j?e5j@e5jAe5jBe5jCjDe5jCjEe5jFe5jGe5jHe5jIe5jJe5jKe5jLhZMeeMZMed#d$ZNd%e e ejOffd&d'ZPd(d)ZQdaRd*d+ZSd,d-ZTd2d/d0ZUdS)3N)contextmanager)partial)CallableUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitioncCs4|jjdtjjjdD]}tjjj|_q ||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor recompile)fx_gnoderI/var/www/auris/lib/python3.10/site-packages/torch/_functorch/compilers.py _canonicalize$s   r!c cs6tjd}z dVWtj|dStj|w)NF)r_CZ_jit_set_autocast_mode)Zold_jit_autocast_flagrrr _disable_jit_autocast-s  r#rreturncCs:tt||jjdtjjjdD]}t|j dkr.t|j dkr.d|j vr.tjjj |_ q|jj D]}i}|j D]\}}t|tjrI|j}|||<q<||_ q3|j|tj|}tj|jtj|}tj|}tdd|Ds||Wd|SWd|S1swY|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. rrr dtypecss|] }t|tjjVqdSN) isinstancerZ _subclassesZ FakeTensor).0trrr `szts_compile..N)r#r rrrrrrlenargskwargsrrnodesitemsr'devicetypeZlintrjitscriptr"Z_jit_pass_remove_mutationfreezeevalZoptimize_for_inferenceany)rinpsrZ new_kwargskvfrrr ts_compile6s<  &          r;TcCst|jt|||d|S)N) clear_meta)printcoder)r_namer<rrr _draw_graph_compilees rAcCsttt|dS)Nr@)r rrArBrrr draw_graph_compileksrCcCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrr?rrr nopos rEcs(eZdZfddZfddZZS)DebugInterpretercs$t|jg|R|_tj|dSr&)rmodulesymbol_mappingsuperrun)selfr, __class__rr rJ}szDebugInterpreter.runc sfddfddfddfdd}t|}d |jvrlt|jd \}}t|\}}t|t|ksJJt|d t|ttt|||D]\}} t| t j s`qT||| fd d qT|S) Ncs:t|ts|St|jjj}|jsJ|t |Sr&) r'rsympyexpandrexprZxreplacerHZ is_numberint)nir)rKrr subst_symints z/DebugInterpreter.run_node..subst_symintcstfdd|DS)Nc3s|]}|VqdSr&r)r(rRrTrr r*szHDebugInterpreter.run_node..subst_symint_tuple..)tuple)ZnisrUrr subst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplecsT|dkr(t|jD]}||||kr'||dkr'dSq dS)Nrr FT)Znumelrangendimstridesize)abidxrUrr check_significant_stridessz.check_significant_stridesc st|sJ|j|jksJ|d|jd|j||ks=J|d|d|d|||}|s\J|d|d|d|dS)Nz:  != z aka )callabler%r[rZ)nvrvZdescZ same_strides)r_rWrr checks ** *z(DebugInterpreter.run_node..checkvalr`csddjS)Nzoutput z where )rHr)irKrr sz+DebugInterpreter.run_node..) rIrun_nodemetapytreeZ tree_flattenr+ziprXr'rTensor) rKnrdrSZn_valsZ_n_specZr_valsZ_r_specrbrcrL)r_rfrKrTrWr rhs    * zDebugInterpreter.run_node)__name__ __module__ __qualname__rJrh __classcell__rrrLr rF|s rFcCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rFrJrDrrr debug_nops rrcCs(t|tj|}tj|}|Sr&)r rr2r3r4r5)rr?r:rrr simple_ts_compiles rscCs t|tSr&)r rs)r:rrr nnc_jits rtcCst|j|Sr&)r=r>rDrrr print_compiles rufncKsFttttd}||t|tjjrt|fi|St |fi|S)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser. Z fw_compilerZ bw_compilerZ partition_fnZdecompositions) r;rdefault_decompositionsupdater'rnnModuler r )rvr-configrrr memory_efficient_fusions r}cCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr)shaper%)r(rfrrr sz!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderr=r~rcudar;)rr7rrrr debug_compiles    rc Csg}t|dU}t|}g}|D]B}t|dkr"|}|t}n,|\}}}} } | tjtjtj tj tjtj tt hvrFtj dd|| | d}ntj|| | d}||qWd|S1s_wY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r%r0N)openpickleloadr+randomZrandrrQZint32Zint64boolZuint8floatrandintappend) Zinput_data_pathZinputsr:Z inputs_metarir1inputrZ_strider%r0rrr get_inputs3s6      rc sbddlm}fddfddfdd}fd d }fd d }||||||td S)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcsg}t|dkr!t|dtr!||d7}||d7}|S|D](}t|tks1t|tkr:|t|fq#|t||j||j |j fq#|S)Nrr ) r+r'rVr1rQrrrrZr%r0)r, input_metaarg)get_input_metarr rdsz(_save_fx_default..get_input_metacst|jjdkrttjd|tdSt|}|j t j j | |}tjddd|ddd|dt t|tddd|dtdd|dtddrt |ddd|dtdd|dtd dSdS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okr?z.inputwbz.pt)r+rr.logloggingWARNING graph_indexcopydeepcopyZ set_codegenrfxZCodeGenrosmakedirsrrdumprsave)Z gm_to_saver, type_namegmr) current_namedump_example_input folder_namerrr graph_saver_helperss< 22z,_save_fx_default..graph_saver_helpercs||d|S)Nforwardr)rZfw_argsrrr graph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)NZbackwardr )r)rZbw_argsrrr graph_saver_backwards z._save_fx_default..graph_saver_backwardcs||dt||S)NZjoint)r)rZ joint_argsrrr graph_saver_joints  z+_save_fx_default..graph_saver_jointrw)Zfunctorch.compilerrx) rrrrZexample_inputsrrrrr)rrrrrr _save_fx_defaultRs   #  rFcCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrrrr graph_dumper_aots r)T)F)Vrrrrr contextlibr functoolsrtypingrrrNrZtorch.fxrZtorch.nnrzZtorch.utils._pytreeutilsZ_pytreerjrZ torch._decomprZ%torch.fx.experimental.symbolic_shapesrZ aot_autogradr r r Z compile_utilsr Z partitionersrrr getLoggerrnrr!r#Z GraphModuler;rArCrEZ InterpreterrFrrrsrtrrdetachZ gelu_backwardZleaky_relu_backwardZsigmoid_backwardZthreshold_backwardZhardtanh_backwardZhardsigmoid_backwardZhardswish_backwardZ tanh_backwardZ silu_backwardZ elu_backwardZcudnn_batch_normZcudnn_batch_norm_backwardZ masked_fillZScalarrlZeluZ leaky_reluZhardtanhZ hardswishZ hardsigmoidZ conj_physicalZ is_same_sizerxrur{r}rrrrrrrrr s           . 7    ']