a h @sUddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Zddl m Z ddl m Z mZddlmZmZmZmZmZddlZddlZddlZddlmZddlmmZddlmZm Z ddl!m"Z"ddl#m$Z%ddl&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-m.Z.dd l/m0Z0m1Z1dd l2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m$Z$ddl@mAZAddlBmCZCmDZDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLddlMmNZNmOZOmPZPerddlQZQe$jRZSeTeUd<eVeWZXejYeUd<ejZj[Z[ejZj\Z\e GdddZ]e GdddZ^e GdddZ_ej`eTd d!d"ZaejbeTd#d$d%ZcejbeTd#d&d'Zdej`eed d(d)ZfGd*d+d+ZgegZhdejiejej`ejej`eekejid,d-d.Zlej`eTd d/d0Zmej`eTd d1d2Znej`eTd d3d4Zoej`eTd d5d6Zpej`eTd d7d8Zqej`eTd d9d:Zrej`eTd d;d<Zsej`eTd d=d>Ztejbeuejej`ejej`fd?d@dAZvejej`ekdBdCdDZweejej`euej`feedEdFdGZxdejjiejj`eyeydJdKdLZzejjiejj`ejj`ej{eyeyejj`dMdNdOZ|ej}eydPdQdRZ~ejej{dSdTdUZejj`eTd dVdWZej{dSdXdYZej{eudZd[d\Zejjidd]d^d_Zejjidd]d`daZdejej`ejbejbeer8r8r8r9r-Cs      r-c@seZdZUeejed<eejed<eejed<eejed<eeje fed<eejed<e j eejddd Z eje d d d Zeje d d dZeje d ddZeje d ddZdS)NodeInfoinputs_required_fw_nodesrequired_bw_nodesunclaimed_nodesfw_orderstatic_lifetime_input_nodesreturncs tddjDfdddS)Ncss|] }|VqdSr5r8.0nr8r8r9 lz-NodeInfo.required_fw_nodes..cs j|Sr5)rKrQr7r8r9lrSz,NodeInfo.required_fw_nodes..key)sortedrHrUr8rUr9required_fw_nodesiszNodeInfo.required_fw_nodes)rQrNcCs ||jvSr5)rHr7rQr8r8r9is_required_fwoszNodeInfo.is_required_fwcCs ||jvSr5)rIr[r8r8r9is_required_bwrszNodeInfo.is_required_bwcCs ||jvSr5)rJr[r8r8r9 is_unclaimeduszNodeInfo.is_unclaimedcCs$||jvsJd|d|j|S)NNode z not in fw nodes!)rHrKr[r8r8r9 get_fw_orderxszNodeInfo.get_fw_orderN)r?r@rAlistrDrErCrdictint functoolscached_propertyrZboolr\r]r^r`r8r8r8r9rF]s rFc@s6eZdZUeed<eed<eed<eed<eed<dS) MinCutOptionsban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)r?r@rArfrCr8r8r8r9rg}s rg)r4rNcCs|jddtjtjfvS)N recompute)metagetrZMUST_RECOMPUTEZPREFER_RECOMPUTEr3r8r8r9must_recomputes rp)fx_grNcCs |jjD]}t|rdSqdS)NTF)graphnodesrprqr4r8r8r9has_recomputable_opss rucCs<|jjD].}t|rt|jdrtjj|jjvrdSqdS)NtagsTF) rrrsrphasattrtargettorchTagnondeterministic_seededrvrtr8r8r9has_recomputable_rng_opss  r|cCs6t|jdtjtjfrdSt|jdtjs2JdS)Nvalr) isinstancernrySymIntSymBoolSymFloatr3r8r8r9 sym_node_sizesrc@seZdZddZdS)InvalidNodeBasecCsdS)Nz Invalid Noder8rUr8r8r9__repr__szInvalidNodeBase.__repr__N)r?r@rArr8r8r8r9rsr) joint_graphrGoutputssubgraphrNc st}i|D] }||j}|j|_||<q|jD]}t|rV|dkrVt|<q8|vrbq8q8|jdkrvt|<q8|jdkrt j |j i|j }fdd|D}t |rt|<q8||fdd|<q8|jdkr||fd d|<q8|jd kr8q8g}|D]h} t| tjrf| vr6td | d t| trVJd | d || n || q |t||||S)a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. backward placeholder call_functioncs&g|]}t|tjrt|tqSr8)rrDrErrPxenvr8r9 s z6_extract_graph_with_inputs_outputs..cs|Sr5r8rrr8r9rVrSz4_extract_graph_with_inputs_outputs..get_attrcs|Sr5r8rrr8r9rVrSoutputr_z couldn't be found in envz was invalid, but is output)rDGraphrnamernrs_must_be_in_backward InvalidNodeoppytreearg_tree_leavesargskwargsany node_copyrrE RuntimeErrorrappendrtupleeliminate_dead_codeZlint) rrGrr new_graphr4new_nodeall_argsZ output_valuesrr8rr9"_extract_graph_with_inputs_outputssV           rcCs,|jdko*dt|jvo*t| o*t| SNrtangents)rstrrx_is_bwd_seed_offset_is_fwd_seed_offsetr3r8r8r9 _is_primals  rcCs|jdkodt|jvSrrrrxr3r8r8r9 _is_tangentsrcCs&|jdko$dt|jvp$dt|jvS)NrZbwd_seedZbwd_base_offsetrr3r8r8r9rs rcCs&|jdko$dt|jvp$dt|jvS)NrZfwd_seedZfwd_base_offsetrr3r8r8r9rs rcCs|jdkot|jdtS)Nrr})rrrnrorr3r8r8r9_is_backward_statesrcCs|jdddkS)Npartitioner_tagZ is_backwardrnror3r8r8r9_has_tag_is_backwardsrcCs|jdddkS)NrZmust_be_in_backwardrr3r8r8r9_has_tag_must_be_in_backwardsrcCst|pt|ot|Sr5)rrr'r3r8r8r9rsr) joint_modulerNcCs>tjdd|jjddD}|d|}||d}||fS)Ncss|] }|jVqdSr5rrPr4r8r8r9rR$rSz+_extract_fwd_bwd_outputs..rr)rrrr find_nodes)rnum_fwd_outputsr fwd_outputs bwd_outputsr8r8r9_extract_fwd_bwd_outputs s   r) saved_valuesrcCs&|D]}|j|kr||q"qdSr5)rremove)rrZ saved_valuer8r8r9_remove_by_name+s  r)fwd_module_outputsrNcCs>t|}tt|dddD]}t||s|d}q:q|S)Nr)lenranger)ridxir8r8r9find_first_sym_node2s  r@-q=)rrr4maxminc Cs\||X|jtjjjj|fd}tjjj|jd|jd<t|jd|jd<Wdn1sh0Y||d|jtjjj j|dgdfd}tjjj |jddgd|jd<t|jd|jd<Wdn1s0Y||`|jtjj j j|tj fd}tjj j |jdtj |jd<t|jd|jd<Wdn1sb0Y||\|jtjjj j||fd}tjjj |jd||jd<t|jd|jd<Wdn1s0Y||X|jtjjjj|fd}tjjj|jd|jd<t|jd|jd<Wdn1sN0Y||\|jtjjjj||fd} tjjj|jd|| jd<t| jd| jd<Wdn1s0Y|| l|jtjj j j| tjfdt|jd} tjj j | jdtj| jd<t| jd| jd<Wdn1sN0Y| S)Nrr} tensor_metarT fp8_scale_rr)inserting_afterrryopsatenabsdefaultrnramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32rr) rrr4rrZabs_nodeZ amax_nodeZ amax_64_nodeZclamp_min_nodeZreciprocal_nodeZmul_node scale_noder8r8r9calculate_quantization_scaling=s  2     2    *    *    *    4     4r)rrr4r quant_typer clamp_maxrNc Csn||`|jtjjjj|tjfd}tjjj|jdtj|jd<t |jd|jd<Wdn1sp0Y||b|jtjj j j ||fd}tjj j |jd|jd|jd<t |jd|jd<Wdn1s0Y||\|jtjj j j||fd}tjj j |jd||jd<t |jd|jd<Wdn1sd0Y||\|jtjj jj||fd} tjj j|jd|| jd<t | jd| jd<Wdn1s0Y|| h|jtjjjj| |fdt|jd} tjjj| jd|| jd<t | jd| jd<Wdn1s`0Y| S)Nrr}r fp8_quant_r)rrryrrrrrrnrrrrrrrr) rrr4rrrrZtarget_node_32Zscaled_target_nodeZclamp_min_scaled_nodeZclamp_max_scaled_nodeZquant_activation_noder8r8r9perform_quantizationsx    (    (    *    *     *r)tensorrNcCs|}|}||dS)z Calculate the size of a PyTorch tensor in megabytes (MB). Args: tensor (torch.Tensor): Input tensor Returns: float: Memory size in MB i)numel element_size)rZ num_elementsrr8r8r9calculate_tensor_sizes rrMcCs.tjjjddd}dd|dD}|S)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16cSs g|]}tt|ddqS).r)getattrrysplit)rPdtyper8r8r9rsz&get_allowed_dtypes..;)ry _inductorrpost_grad_fusion_optionsror)rr8r8r9get_allowed_dtypessrcCst}t|r|jdj|vr"dStjjjddd}t |jd}tjjjdddsd||kStjjjdddrt ||kpt ||k St ||kSdS)Nr}Fr size_in_mbdZskip_dynamo_guardsZquantize_dynamic_shape) rr rnrryrrrrorrr)r4rZsize_thresholdrr8r8r9should_quantizes4 rcCs*tjjjddd}tt|ddS)Nrrztorch.float8_e5m2rr)ryrrrrorr)rr8r8r9get_quant_types r)rrNcCst|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )ryZfinforr)rinfor8r8r9calculate_range s r)rrrNc s|jddd}|jd}t}t|\}}tgg}}|D]}|jddr@tjj j dddrt |||d } t ||| |||} t | s|| n || n||h|jtjjjj||fd t|jd } tjjj|jd || jd <t| jd | jd <Wdn1s"0Y| |<q@fdd|D} t| } ||} | rz| d| | | | d} |dt| tddd7<dS)Nrrrsaved_for_quantizationFr use_scalingTrrrr}rcs g|]}|vr|n|qSr8r8rZ node_to_quantr8r9rDsz*quantize_activation_fw..inductorZ%activation_quantization_fwd_aten_passr)rrrrrbrnroryrrrrrrrrrrrrrrrrrZ update_argrr )rrrrrrrZtensor_scale_nodesZsym_scale_nodesr4rZ quant_nodeZoutput_updated_argsrZ scale_nodesr8rr9quantize_activation_fws`           *  rc sdd|jD}d}|D]}|jddr|jd|jd}tjjjdddr||8d|j d d t fd d |D}Wdn1s0Y||\|j tj jjj||fd }tj jj|jd||jd<t|jd|jd<Wdn1s0Y||b|j tj jjj||fd }tj jj|jd|jd|jd<t|jd|jd<Wdn1s0Y||\|j tj jjj||fd }tj jj|jd||jd<t|jd|jd<Wdn1s0Yn||h|j tj jjj||fdt|j d}tj jj|jd||jd<t|jd|jd<Wdn1s0Yt|jD]&}||kr||kr|||qqtddd7<dS)NcSsg|]}|jdkr|qS)rrrr8r8r9rTrSz*quantize_activation_bw..rF dequant_typerrrrc3s|]}|jkr|VqdSr5r)rP bwd_inputZ scale_namer8r9rRbs z)quantize_activation_bw..rr}rZdequant_rrZ%activation_quantization_bwd_aten_passr)rsrnropopryrrrrrrnextrrrrrrrdivrrrauserskeysZreplace_input_withr ) rrZ bw_inputsZactivation_noder4rrZdivided_target_node_32Z dequant_nodeuserr8rr9quantize_activation_bwSs     &    *     "    ,     *r)r fwd_module bwd_modulerLrNc s tjdddurdS|r(dd|Dng}dd|D}tjjjdddr^dd|D}jjd d d jd }d djjd d D}|D]t}|j |vrt |r|j |vrt d|j qd|j d<|j dj|j d<d||j j d<|j dj||j j d<qtdddfdddtjtdddfdddtdddfdddjjd d d jd } | D]} d| j vr||| j dd} j|  jj| j d} Wdn1s0Y| j d} | j | j d| j d<| | j d<| | j| q|tjjjdddrtjjd d }|d }t|D]}t|s\|}qxq\jjd d d jd }|D]b} d!| j vrj| jj| j d}Wdn1s0Y|j | j |}qtjtdd"dfd#dddS)$NrcSsg|] }|jqSr8rrr8r8r9rrSz2enable_activation_quantization..cSsi|] }|j|qSr8rrr8r8r9 rSz2enable_activation_quantization..Zexclude_primalsFcSsi|]}d|jvr|j|qS)Zprimalsrrr8r8r9r srrrcSsi|] }|j|qSr8rrr8r8r9r srz*Skipping quantization of static input %s: Trr}rZartifactcSs dddS)NZ,before_activation_quantization_fwd_aten_passstringrencodingr8r8r8r8r9rVsz0enable_activation_quantization..csjddddSNFT)Z print_outputZinclude_strideZinclude_deviceZprint_readabler8rr8r9rVs)Z metadata_fnZ payload_fncSs dddS)NZ+after_activation_quantization_fwd_aten_passr r r8r8r8r8r9rVscsjddddSr rr8rr8r9rVscSs dddS)NZ,before_activation_quantization_bwd_aten_passr r r8r8r8r8r9rVscsjddddSr rr8rr8r9rVsrrrrrrcSs dddS)NZ+after_activation_quantization_bwd_aten_passr r r8r8r8r8r9rVscsjddddSr rr8rr8r9rVs)inductor_configrroryrrrrrrrrr,debugrnrrrrrrupdatereplace_all_uses_with erase_noderareversedrr)rrrrLZstatic_input_namessaved_values_namesrZbwd_module_inputsr4Zquant_fwd_module_outputsZfwd_noderZquant_bwd_inputrZquant_bwd_module_inputsZ bwd_input_locZbw_inputZscaled_fwd_module_outputsZscale_bwd_inputr8)rrr9enable_activation_quantizations          0        0  r)rL)rrsaved_sym_nodesrrLrNcCsTt||d\}}|jjdd}gtt|}gtt|} gtt|} gtt|} gtt|} t |j||| | |d} t j }| jddD]r}|j st||jt||jq|rtdd|j Drt||jt||jqt|rt||j| sJqt}g}g}|D]4}t|}|rD||||n ||qt|j}t||| D]d}d|jvr~qjt|jd|}t|dd d D]"}||vrq|||q||O}qj||||t |j|| |||d }t |j||| | | |d} tj||}tj|| }t ||||||fS) Nrrrrcss.|]&}|jtjjjjuo$t|jdkVqdSrN)rxryr_c10d_functionalZ wait_tensorrrrrOr8r8r9rRCsz+_extract_fwd_bwd_modules..r}cSs|jSr5r)sr8r8r9rVirSz*_extract_fwd_bwd_modules..rWforward)!rrrrfilterrrrrrrry distributed is_availablerrrallrraddrr itertoolschainrnrrYclearextendrDZ_lazy_graph_moduleZ_make_graph_moduler)rrrrrLrrZ placeholders primal_inputstangent_inputsfwd_seed_offset_inputsZbwd_seed_offset_inputsZbackward_state_inputsZ bwd_graphZdistributed_enabledr4Z saved_symbolsZsaved_sym_nodes_bindingZsaved_sym_nodes_derivedsymbolZsymbol_bindingsZ new_symbolsrZ fwd_graphrrr8r8r9_extract_fwd_bwd_moduless               r,)static_lifetime_input_indicesrL)rr-rLrNcsjt|rt||||dSttt|jj}ttt|jj}||}t||d\}} t |j||d} t dd| jDg} g} |jjD]} | j vrqt | r| | qd| jvr| jdkr| j}tdd|DsJ| |qfd d | jD}d| jvr(td d|Dr(| |q| | qtt| } tt| } t|| | ||d S) a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. )rr-rrcss|]}|jdkr|jVqdSrNrrrr8r8r9rRsz$default_partition..rrcss|]}|jtjkVqdSr5)rxoperatorgetitemrPrr8r8r9rRrScsg|]}|jvr|qSr8rrOZforward_node_namesr8r9rsz%default_partition..css|]}t|VqdSr5rrOr8r8r9rRsrrrL)ru#min_cut_rematerialization_partitionrarrrrrsrrrrrrrrnrrr"r'rbfromkeysrr,)r _joint_inputsrr-rLr(r*rGrrforward_only_graphrrr4rZbackward_usagesr8r3r9default_partitions^         r:g.A)rrNcCs ||jSr5)itemsize)rrr8r8r9_tensor_nbytessr<cstdddd|jvr|jd}t|tr0dSt|ttfrTtfdd|DSt|trxtfdd|DSt|t j r|St d t |d ||j d ks|jt jjjjurd St d |ddS)NrMcSs(t|tjsdStt|dd|jS)Nrfallback)rryrr<rrrrr8r8r9 object_nbytess z_size_of..object_nbytesr}rc3s|]}|VqdSr5r8rOr@r8r9rRrSz_size_of..c3s|]\}}|VqdSr5r8)rP_rQrAr8r9rR rSzUnknown metadata type z on node rrr_zO didn't have `val` metadata; we should always have `val` metadata on the nodes.)rcrnrrrarsumrbitemsryrrtyperrxrrZ_assert_scalarr)r4r}r8rAr9_size_ofs"      rF)rrcCsbddlm}|t}|jD]"}|jdkr||jjd7<qtdt | t ddddS)Nrrrrz%sTrXreverse) collectionsrrcrsrrxr?r,rrYrDr0 itemgetter)rrrZcntr4r8r8r9 _count_opss    rKcCslg}ttjjD]V}ttjj|}t|tjjs2q|D]*}t||}tj j |j vr:| |qq:q|Sr5) dirryrrrr_opsZOpOverloadPacketZ overloadsrzZ pointwiservr)r attr_nameZopoverloadpacketoverloadZ op_overloadr8r8r9 pointwise_ops s   rP) depth_maprNcs*fdd|D}t|tdddS)Ncs&i|]}t|tjjjr||qSr8)rryrDr4rE)rPargrQr8r9r 3szsort_depths..rTrG)rYrDr0rJ)rrQZ arg_depthsr8rSr9 sort_depths2s rT)gmrNc s0ti|jjddD]}|fdd|<qddt|jjDfdd}ttt |jj}d }t j }|D](}|j D]}||kr|}|}qq~|d ur|St|jjd |D](}|j d kr|jtjjjjkr||qt|jj|d D]}||qtj|}|S) a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traveral, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rrcs|Sr5r8rrr8r9rVUrSz5reordering_to_mimic_autograd_engine..cSsi|]\}}||qSr8r8rPrr4r8r8r9r WrSz7reordering_to_mimic_autograd_engine..cs|g}t}t|dkrH|}||vs |vr2q ||||j7}q t|fddd}|D]}|fdd|<q`dS)Nrcs|Sr5r8rT)orderr8r9rVfrSzSreordering_to_mimic_autograd_engine..insert_node_in_graph..rWcs|Sr5r8rrr8r9rVhrS)rrrr#Zall_input_nodesrYr)r4 cur_nodesZinsertable_nodesrrrWr8r9insert_node_in_graphYs   zAreordering_to_mimic_autograd_engine..insert_node_in_graphNr)rDrrrrr enumeratersrarrmathinfrrrxryrrZcopy_r GraphModule) rUr4rZr)Zfirst_node_in_bwdZ minimum_ordertangentrZnew_gmr8rYr9#reordering_to_mimic_autograd_engine9s.    r`) fw_module bw_modulefw_nodebw_nodedevice rng_countlast_fwd_inputlast_bwd_inputcCs|j}|dusJ|j} |j} tjjj} |j|4|jd|} t|| j d<| }Wdn1sn0Y|j|4|jd|} t|| j d<| }Wdn1s0Yt |j }| |d<|j|.| j d| |j g|jR|d}Wdn1s0Y||| |t |j }| |d<| |B| j d| |j g|jR|d}||| |Wdn1s0Y||fS)a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 NZfwd_rng_state_r}Zbwd_rng_state_Z rng_staterrr)indexrrry_prims rng_primsgraphsafe_run_with_rng_staterrr&rnrbr create_noderxrrrinserting_before)rarbrcrdrerfrgrhZ device_idxfw_graphbw_graphrmZ fwd_rng_stateZ bwd_rng_stateZ fw_kwargsfunctional_fw_nodeZ bwd_kwargs rng_outputr8r8r9%apply_graphsafe_rng_functionalizationsH)  "" &     *rt)rrarb num_sym_nodesrNc' st}dd}ttjdddttjddd}||}||}||} i} |jjD]T} t| r\t| j d r\tj j | j j vr\|| j } || j } | | j }| |d | | <q\tjjj}tjjj}d}|jjd d D]} d | j vr| }qq|durtdg}tt|jjd d }tt|jjd d }tfdd| D}|tdt|dk}tjj}tjo| o|j p|jj}t | !D]\}\} }|d} |d}| }|j}|j}|r |dur |j"dkr t#||| |||||\}}q|$| ~|j%d|| j g| j&R| j'd}|j%dt(j)|dfid}|j%dt(j)|dfid} | *| |+| |,|Wdn1s0Y|$|6dt|}!|-|!}"|||"j.d<Wdn1s0Y|$|F|j%d||"|j g|j&R|j'd} |*| |+|Wdn1sN0Yq|rtt/|jjdd }#|#j&d}$t|$|}%|$d|%t0||$|%d}&|j1|&|j+|#|2|2||fS)NcSsFi}|jjD]4}|jdkr t|jdr tjj|jjvr |||j <q |S)Nrrv) rrrsrrwrxryrzr{rvr)ZgmodZ random_nodesr4r8r8r9 get_rng_opss   z*functionalize_rng_ops..get_rng_opsrMcSs^d|jvrdS|jd}t|ts(|f}|D]&}t|tjr,|jjdkr,|jSq,tdS)zV Check the example value of the node outputs to find the device type. r}Ncudacpu)rnrrryrrerE)r4 candidates candidater8r8r9 get_device s      z)functionalize_rng_ops..get_devicerecSs$|dur|jdkrtjStS)Nrw)rEryrwZ get_rng_stater|r8r8r9get_sample_rng_states z3functionalize_rng_ops..get_sample_rng_staterv)fwdbwdrrr_zaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc3s|]}|dVqdS)r~Nr8)rP node_pairr{r8r9rRBsz(functionalize_rng_ops..rxrr~rrwrrirZrng_state_output_r}r)3r$countrryrerrrsrprwrxrzr{rvrrkrlZrun_and_save_rng_staterun_with_rng_staterrrrrvaluesdiscardrrrZgraphsafe_rng_functionalizationZfallback_randomZ test_configsZ*graphsafe_rng_func_ignores_fallback_randomr[rDrErtrornrrr0r1rrrrrniterrr recompile)'rrarbruuidrvr}Zjoint_graph_rng_opsZfw_graph_rng_opsZbw_graph_rng_opsZrecomputable_rng_ops_mapr4Z base_nodercrdZrun_and_save_rngrZbw_tangent_start_nodeZfw_rng_state_outputsrgrhZdevicesZmulti_cuda_devicesZ ind_configZ'use_rng_graphsafe_rng_functionalizationrfrrerprqrrstatersZ state_nameZbw_rng_state_nodeZfw_output_nodeZ fw_outputsZsym_node_start_idxrr8rr9functionalize_rng_opss                 *  .  .      rcCsB|jjD]4}t|jtjjr|jjdkrt|st j |j d<qdS)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrmN) rrrsrrxryrM OpOverload namespacerpr MUST_SAVErn)rr4r8r8r9force_save_collectivess  rcCs||jjD]n}t|r|jD],}t|r|jd|jdkrtj|jd<q|jddrtdd|jDstj|jd<q|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. Z ac_graph_idrmZhas_backward_hookFcss|]}t|VqdSr5)rpr2r8r8r9rRsz)cleanup_recompute_tags..) rrrsrprrnrrror)rr4rr8r8r9cleanup_recompute_tagss  r)r node_infomin_cut_optionsdont_banc$ sdurtt trPtdd|jD}|tdd jD}td|dddd fd d z d dl}Wn.ty}zt d |WYd}~n d}~00 fd d fdd}fddt d fdd } | t fdd} |jD]} | j dkr(q| j vrt| jvr\ j| jddtjdq j| jddtjdt| r j| jddtjdqt| st| r| |  | r|| r| | d| jvrd| jvp d| jvo t| jdtj } t| r$t t| } n2| rJt| jdtrBdntj} n | | j} j| jd| jd| d| j D]$} j| jd|jdtjdqxqt!t"j#t$t$d  fd!d" }j%r j&D]} fd#d$|j D} fd%d$|j D}t'|d kr||t(|}t)|j D]d} |r *||kr ||r |vrZq td&| *||| *|| |q qΈj+rt}|jD]} |sq *||fg} *|}t'|d krt,-|\}}||vrq|.| *||d'krLt'|d krLtd(|| *| *|| |q|j D]>} |rR||rR|vrRt,/| *||fqRq̐qz|0 d)d\}}WnBt1ytd*td+2|j3j45 t6 Yn0|\} t} fd,d|DD]$\}|7 fd-d|Dqt}|D]>\} }!| dd.|!dd/ksjJ| dd.}"|.|"qDt8|d0d1t9|jD t:fd2d|D fd3d4d5}#|#fS)6Ncss0|](}|jdkrt|jdrt|jjVqdS)r_overloadpacketN)rrwrxrrrr8r8r9rRsz solve_min_cut..css|]}t|VqdSr5)rrPrr8r8r9rRsz&Ops banned from re-materialization: %scSsn|jtjjjkrdS|jd}tjj|\}}|D]4}|j |}||urPdSt |t r4||vr4dSq4dSNFrT) rxryr higher_orderZauto_functionalizedrZ_higher_order_opsZauto_functionalizeZget_mutable_argsrrra)abZ mutable_opmutable_arg_namesrBrrRr8r8r9!can_fuse_into_auto_functionalizeds    z8solve_min_cut..can_fuse_into_auto_functionalizedcSsH|jtjjjkrdS|jd}|D] }|jd|}||ur"dSq"dS)NFZtensors_to_clonerT)rxryrr triton_kernel_wrapper_functionalr)rrrrrRr8r8r9.can_fuse_into_triton_kernel_wrapper_functional s zEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functionalcsht|tjkrdS||r dS||r.dS|jtjurT|jdjtjj j urTdS |of |S)NTrF) r)rcatrxr0r1rryrrrr:)rr)rrop_typesr8r9r:s    z!solve_min_cut..is_fusiblerzANeed networkx installed to perform smart recomputation heuristicscsl|rdSt|g}t|dkrh|}|jD]2}|sP||sPdS|r2||q2qdSr)r=rrrrr\r#)r4rXcurr)r:rrr8r9is_materialized_backwards-s     z0solve_min_cut..is_materialized_backwardscs|jdkrdS|jtjkrdS|jddtjkr6dStj rJ |rJdS|jt j j t jj fvrddSjrz|sdSn|s|rdSjr|rtd|t|jdS|jdkr|jtjkrdSjrtdd|jD}t|}|d |kSdS) NrFrmTzmaterialized backwards: %s %sicss"|]}t|tjrt|VqdSr5)rrDrErFrr8r8r9rRfszBsolve_min_cut..should_ban_recomputation..r~)rrxr0r1rnrorrrrecompute_viewsr=rlift_fresh_copyrZ lift_freshrkr>r<r;rjr,rrr dist_from_bwZmax_dist_from_bwrlrCrrF)r4Zinput_tensors_sizeZ output_size)rrrr8r9should_ban_recomputation;s:    z/solve_min_cut..should_ban_recomputationcs*jdkrdStfddjD S)NrTc3s|]}|VqdSr5r8r2)r:r4r8r9rRqrSz9solve_min_cut..is_materialized..)rr"rr3)r:r3r9is_materializedms z&solve_min_cut..is_materializedrMcstjr||vrdSt|}tjr0|r0tjSt|jdt rVt|jdt j sVt St |dtt|jdd}|r|S|dSdS)Nrr}g?rr)rZ treat_parameters_as_free_to_saverFrr=r\r]rrnrryrINT_INFrcrrr)r4rLZmem_sz)rrr8r9get_node_weightssz&solve_min_cut..get_node_weightcs|rdS|vr@t|jtjjo0|jjdk}tjs<|s@dSt |rLdSd|j vrlt|j dtj rldS |j d|jdtjddS)NFrr}source_incapacityT)r=rrxryrMrrr(unsafe_allow_optimization_of_collectivesrprnrr#add_edgerr\r])r4Z is_collective) banned_nodesrnx_graphrr8r9ban_recomputation_if_alloweds    z3solve_min_cut..ban_recomputation_if_allowedrrZsinkrZ_outr}r) start_nodes max_rangerNc sg}|D]}t|||dfqt|dkrt|\}}}|sP|S|jD]H}|rV||krtqV||||f}||vrVt||qVq&|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushr`rheappoprr\) rrZ sorted_nodesrQrBr4Znode_is_fusiblerr})r:rr8r9find_first_unfusibles$    z+solve_min_cut..find_first_unfusiblecs g|]}|r|qSr8)r\r`r2rr8r9rs z!solve_min_cut..csg|]}|r|qSr8)r\r2rr8r9rsz1used above/below fusible %s:(%s) -> %s -> %s:(%s)rztoo long %s %s %s %srz-Failed to compute min-cut on following graph: c3s|]}||fVqdSr5r8rO)rr8r9rRgrSc3s|]}|vr|fVqdSr5r8)rPv) non_reachableur8r9rRhrScSsi|]\}}||qSr8r8rVr8r8r9r rrSz!solve_min_cut..c3s|]}|VqdSr5r8r name_to_noder8r9rRtrScs|Sr5r8r)node_idxr8r9rVtrSzsolve_min_cut..rW);rget_default_op_listr+rsr2r,rnetworkx ImportErrorrfloatZDiGraphrrIrGrrr\r]rprrr\rnrryrrrrorrLrrarDrErcrhrZrrrr`rirrr#rZ minimum_cut ExceptionjoinZ readwriteZedgelistZgenerate_edgelistvisualize_min_cut_graphrget_name_to_noder[rY)$rrrrZjoint_module_opsZ ops_ignorednxerrrr4Zis_non_tensor_nodeweightrrZ used_nodeZordersZfw_usersZfirst_unfusible_usevisitedZ start_nodeZfusibleZ start_orderrBrZ cut_value partitionZ reachableZcutsetZnbrsZ cut_nodesZnode_inZnode_outZ node_namerr8)rrrrr:rrrrrrrrrrr9 solve_min_cuts     2          &              "   rcCsddl}ddl}|j|}||d}|D]@}|||d}| t ||t dkr6| dq6t d|ddS)Nrrr]Zredz2Visualizing the failed graph to min_cut_failed.svgzmin_cut_failed.svg)rpydotZnx_pydotZto_pydotZ to_stringZgraph_from_dot_dataZ get_edges get_sourceZget_destinationZ set_labelrrZ set_colorr,rZ write_svg)rrrZ dot_formatZ dot_graphZedgerr8r8r9rys    rcKCstjtjtjtjtjtjtjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtj tj!tj"tj#tj$tj%tj&tj'tj(tj)tj*tj+tj,tj-tj.tj/tj0tj1tj2tj3tj4tj5tj6tj7tj8tj9tj:tj;tjtj?tj@tjAtjBtjCtjDtjEtjFtGjHtjItjJtjKtjLgK}tjItjJtjMg}|tjNtjOtjPtQjRtjStjTtjUtjVtjWg 7}|}|tQjtQjXtjYtjLtjZtQj[tQj@tj[tj\tQjRtjVtj]tjNtjStjOtj^tj_tj`tjatjbtjctjdtjetjftjgtjhtjitjTtjjtjktjltjmtjntQjotQjpg#7}|tjqtjrg7}||7}|ts7}|tjtg7}|ddtuD7}tv|}tvtwdtxftjytjztj{g}tj|tj}tj~tjtjtjtjtjtjtjtjg }||B}t|tv||tv||S)NcSsg|] }t|qSr8)r)rPmr8r8r9rrSz'get_default_op_list...)rr#subratan2rrrpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltrZ bitwise_notceilfloorfracnegZreluroundZsilutruncr,log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtZrsqrtrZsigmoidZsoftplus thresholdZthreshold_backwardclampwhereZlerpZaddcmulZgeluZ gelu_backwardrCmeanZ_grad_sum_to_sizeZ sum_to_sizertoZtype_asr0r1ZsqueezeZ unsqueezeZrsubZ_to_copyaliasviewslicetrZbroadcast_in_dimexpandZ as_stridedZpermuteselectrrcloneZ full_likevarZstdZ _unsafe_viewZreshapeZbroadcast_tensorsZ scalar_tensorZonesZ new_zerosrZarangeZtriuZvar_meanisinfrfullzerosemptyZ empty_likeZargmaxmaximumiotaZ'_low_memory_max_pool_offsets_to_indicesrjZgatherrPZ zeros_likerrrrZnative_dropoutZ rand_likeZ randn_likemmZ convolutionZconvolution_backwardZbmmZaddmmZ#_scaled_dot_product_flash_attentionZ'_scaled_dot_product_efficient_attentionZ_flash_attention_forwardZ_efficient_attention_forwardZupsample_bilinear2dZ _scaled_mmr-)Zdefault_recomputable_opsZrecomputable_view_opsr1r2r0r/r.r8r8r9rs:M & rcCsi}|jD]}|||j<q |Sr5)rsr)rrrr4r8r8r9r2s  r)rmemoryruntimes max_memoryrall_recomputable_banned_nodesrNc Cstj}|dkrt|||S|dkr.t|||S|dkrBt|||S|dkrtdtj||||d}t||t |dj t|dSt |r||||||\}} d || fSt d |dS) NZgreedyZilpZdpZdynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rrZ recorded_knapsack_input_memoriesZ recorded_knapsack_input_runtimes)graph_info_provider)Z knapsack_algoZmax_mem_budgetrz,Not aware of memory budget knapsack solver: ) rZactivation_memory_budget_solverr"r#r!r,warningr Zinialize_from_graphr$Zget_knee_point_memory_budgetcallabler) rrrrrrZSOLVERrZsaved_node_idxZrecomp_node_idxr8r8r9#_optimize_runtime_with_given_memory9sB      r no_dispatch)rr?rNcsLt|j}fddfdd|D}fdd|D}|j||dS)Ncs t|dS)Nr>)r)dr>r8r9realize_symbolnsz8_remove_symbols_without_guarding..realize_symbolcsg|] }|qSr8r8rPrrr8r9rqrSz4_remove_symbols_without_guarding..csg|] }|qSr8r8rrr8r9rrrS)stride)rashaperZnew_empty_strided)rr?r rr8)r?rr9 _remove_symbols_without_guardingks   r!cstj}dd}|dkrdS|dkrtNddlm}t|jjf\| fdd }|WdS1s|0Yn|d krdd l m }t|jjf\|d d  }j iWdn1s0Y| }t|dStd|dS)NcSst|tjr0t|jdtjr0t|jdddSt|tjr`t|jdtjr`t|jdddSt|tjrt|jdtj rdSt|tjrt|jdtj rdS|SdS)Nr}r=r>?T) rrDrErnryrr!rrrrrr8r8r9materialize_argysz)estimate_runtime..materialize_argZtestingrZprofiler) benchmarkercsjiSr5)rxr8rrr4r8r9rVrSz"estimate_runtime..Zflops)FlopCounterModeF)Zdisplayz Not aware of runtime estimator: )rZ*activation_memory_budget_runtime_estimatorrZ$torch._inductor.runtime.benchmarkingr$rZtree_maprrZ benchmark_gpuZtorch.utils.flop_counterr&rxZget_total_flopsrr)r4Z RUNTIME_MODEr#r$msr&modeZ counted_flopsr8r%r9estimate_runtimevs$  $   . r))rrrNcsl|dks|dkrtd|ttjtjtjtjtjd}tjrRt |ddddd}|dkr`j St |\}}|dkr||St t jtddd j  | kr|S fd d t t jd  fd d t |dddd}t |\}}||kr |St |ddt \}} ||kr:|Sddlmtfddj Dtt jt t jdfdd } | | } dd| D fdd| D} t| tddtdkrΈj SfddD ddDddlm  fdd tjrZfd!d"} | d#| d$g} | ddd| dddkr| d| dfg}|r|\}}|d|dd%kr| || |qn| |d|dd&}|dd|ddkr|||f|dd|ddkrn|||fqn| ddlm}d'd| D}d(d| D}|jd)d*|j||d+d,t |D]*\}}|j!|d-|||fd.d/d0d1qn|"d2|#d3|$d4|%d|&}|'t()}tj*durtj*}t(j+|dd5d6}t,j-.r&t,j-/r&d7t,j-0}t(j12|d8|d9t3d:}|4|t56d;||d<dS)=NrrzJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )rhrirjrkrlF)rhrirjrk)rrNcSsttt|dSNeA)rCmaprF)rr8r8r9estimate_activations_sizesz:choose_saved_values_set..estimate_activations_sizecs|dSr*r8)sz) max_act_size min_act_sizer8r9get_normalized_sizesz4choose_saved_values_set..get_normalized_sizeZ activationscs|Sr5r8r2)r-r/r0r8r9 get_mem_ratios z.choose_saved_values_set..get_mem_ratio)rhrirj)rkget_node_storagec3s|]}|VqdSr5r8rr4r8r9rRrSz*choose_saved_values_set..)rrNcsfdd|DS)Ncs*g|]"}|jtdkr|vr|qS)r+)rrcrr5input_storagesr8r9rs zRchoose_saved_values_set..get_recomputable_banned_nodes..r8)rr6r8r9get_recomputable_banned_nodess z>choose_saved_values_set..get_recomputable_banned_nodescSs$g|]}|jddtjkr|qS)rmF)rnrorrrr8r8r9rsz+choose_saved_values_set..csg|]}|vr|qSr8r8r)must_save_nodesr8r9r sTrGcsg|]}t|qSr8rFr)r1r8r9r scSsg|] }t|qSr8)r)rr8r8r9r src s.t|t|d|\}}}Wdn1s:0Yt}|D]*}z||WqNtyvYqN0qN|sJt|||\}} trt|||||d||fS)Nr)rrsaved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodesruntimes_banned_nodesZmin_cut_saved_values) rrrr# BaseExceptionissubsetrr+r ) memory_budgetrrr=r;r<rrrrB)aggressive_optionsrr>rr?r8r9get_saved_values_knapsack sN   z:choose_saved_values_set..get_saved_values_knapsackcs(|d\}}|t||fS)N)rr)rC)rrr=)r3rDrrr?r8r9estimate_for_budgetC s  z4choose_saved_values_set..estimate_for_budgetrr"gMbP?rcSsg|] }|dqS)rr8rPitemr8r8r9r` rScSsg|] }|dqS)rr8rFr8r8r9ra rS) )Zfigsizeo)markerz.4fz offset points)rrHcenter)Z textcoordsZxytextZhaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okrZ_rank_Zmemory_budget_paretorBz.svgz%Generated Pareto frontier curve at %s)rBrr)7rrgrZban_recompute_used_far_apartZ!ban_recompute_long_fusible_chainsZ#ban_recompute_materialized_backwardZban_recompute_not_in_allowlistZban_recompute_reductionsZaggressive_recomputationrrGrrarDrErZtorch._inductor.fx_utilsr5rrYrFrtorch.utils._mode_utilsrZvisualize_memory_budget_paretorrsortZmatplotlib.pyplotZpyplotfigureZplotr[ZannotateZxlabelZylabeltitleZgridZgcfshowosgetcwdZmemory_budget_pareto_dirmakedirsryr r!is_initializedZget_rankpathrr%Zsavefigr,r)rrrBrZruntime_optimized_saved_valuesrBZmore_aggressive_optionsZmore_aggressive_saved_valuesZ%aggressive_recomputation_saved_valuesrr8Zrecomputable_banned_nodesrEoptionsZbisectslhsrhsZmidZpltZx_valuesZy_valuesrtxtZfigZfig_dirZ rank_suffixZfig_namer8)rCrr-r3r5r1rDr7rr/r>r0r9rrr?r9choose_saved_values_sets        + "              r\)rrc sddlmdd}fdd}tjrtjrtjdkr||r||rtxRdd |Dg}tjj|dd |d}t |fd d |D}Wdn1s0YWdn1s0Y|S) Nrunset_fake_temporarilycSs2|jD]&}t|jtjjr|jjdvrdSqdS)N>rZc10d_functionalTF)rsrrxryrMrr)rr4r8r8r9has_collectives s   z2_broadcast_rank0_decision..has_collectivesc sddd|jD}t|d}ddttj Dt Dtj |Wdn1st0YWdn1s0Yt fddDS)N/css|] }|jVqdSr5rrr8r8r9rR rSzD_broadcast_rank0_decision..has_same_nodes..zutf-8cSsg|]}dqSr5r8)rPrBr8r8r9r rSzE_broadcast_rank0_decision..has_same_nodes..c3s|]}d|kVqdSrr8rZ all_inputsr8r9rR rS) rrshashlibsha256encode hexdigestrryr get_world_sizerZall_gather_objectr")rZnode_strrGr]rar9has_same_nodes s Jz1_broadcast_rank0_decision..has_same_nodesrcSsg|] }|jqSr8rrr8r8r9r rSz-_broadcast_rank0_decision..)srccsg|] }|qSr8r8rOrr8r9r rS) torch._subclasses.fake_tensorr^ryr r!rVrfrZbroadcast_object_listr)rrr_rgZobjectsrr8)rr^r9_broadcast_rank0_decision s(   Nrjrr-)rr-rNcs|j||j}tjr,t|}||_|j}t|}t|} |rNt|}tj s\t |fdd} |durtg}| ||} t | j dkrt |||| jdSt|jjD]V} | jdkrtd| _q| | sd| _qtd| _| jD]} t| j| jd| _qqtj}|jD],} t| jd dtr| jd }q>qt|| |d }tjr^t||}ttt |}ttd d |}t!|||| jd \}}|r| rt"|||t |\}}t#|}t$|}t$|}t%rt&dd|D}t'dd|Dd}t()d|t()d|t*dd|jjD}t*dd|jjD}||@}t+t}|jjD]8} | j,|vrZt-| j.drZ|t/| j.j0d7<qZt()dt |t |t |t&|1t23ddd}t()d|||fS)ax Partitions the joint graph such that the backward recomputes the forward. Recomputing helps in trading off memory bandwidth with computation. To create the fwd and bwd graph, we copy the joint graph, manually set the outputs to just original forward or backward outputs. And then we run the resulting graphs through dead code elimination. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. _joint_inputs: The inputs to the joint graph. This is unused. compiler: This option determines the default set of recomputable ops. Currently, there are two options: ``nvfuser`` and ``inductor``. recomputable_ops: This is an optional set of recomputable ops. If this is not None, then this set of ops will be used instead of the default set of ops. num_fwd_outputs: The number of outputs from the forward graph. Returns: Returns the generated forward and backward Fx graph modules. c sVt|jt|jjD]J}|jdkr.classify_nodes..rc3s"|]}|jdkr|jVqdSr.r/rrr8r9rR s c3s"|]}|vr|vr|VqdSr5r8r)rIrZr8r9rR sc3s|]\}}|vr|VqdSr5r8)rPrprkr8r9rR srr)rrrrrsrrxr#rrrrarrrrrr[rF) rr-r4r(r*rGrrr9rJrLZfw_cntrKr)rrIrZr-r9classify_nodes s\             z;min_cut_rematerialization_partition..classify_nodesNr)rr-rLrr+rrB)rBcSs t| Sr5r4rTr8r8r9rVM rSz5min_cut_rematerialization_partition..r5cSsg|]}t|t|fqSr8)rFrrr8r8r9re rSz7min_cut_rematerialization_partition..css|]}t|VqdSr5r:rr8r8r9rRh rSz6min_cut_rematerialization_partition..z'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %scss|]}|jdkr|jVqdSrNr/rr8r8r9rRm scss|]}|jdkr|jVqdSrnr/rr8r8r9rRp srz# remat/fw/bw: %d/%d/%dTrGzCount of Ops Rematerialized: %s)4rrrrrZcser(rur|rrrrrIr:rLrrsrrcrr\rrZactivation_memory_budgetrrnrorr\rjrarrr,rr`r*r+rYrCr,rrrrrwrxrrrDr0rJ)rr8compilerrr-rqZ cse_graphrZgraph_has_recomputable_opsZgraph_has_recomputable_rng_opsrmrr4rrBrrrarbZ sorted_sizesZtotal_activations_size_gbZfw_module_nodesZbw_module_nodesZ remat_nodescountsZrematerialized_opsr8rr9r6 s"  6                r6fx_graphTF)tracedfnamefigname clear_metaprogparse_stack_tracedot_graph_shaperNcCs|r0t|j}t||}|jjD] }i|_q$tj |\} } | sNdt j } t d| | tj||||d} | } t| d| d} | | }|dur| |n | ||ddS)NrzWriting FX graph to file: %s%s)rwrxZwrite_)rv)copydeepcopyrrrDr^rsrnrSrWsplitextrZtorch_compile_graph_formatr,rrZ FxGraphDrawerZget_main_dot_graphrlstrip)rrrsrtrurvrwrxrr4baseextgrZ write_methodr8r8r9 draw_graph s*       r)N)rr)N)N)r)r)rqTNFN)ryrdrbrr$loggingr\r0rSos.pathrIrZ dataclassesrrtypingrrrr r ryZtorch._inductor.inductor_primsZtorch.distributedZtorch.fxrDZtorch.utils._pytreeutilsZ_pytreerZtorch._dynamo.utilsr r Z;torch._functorch._activation_checkpointing.ac_logging_utilsr Ztorch._inductorrrZtorch._loggingrrirZ%torch.fx.experimental._backward_staterZ"torch.fx.experimental.proxy_tensorrrZtorch.fx.experimental.sym_noderrZ%torch.fx.experimental.symbolic_shapesrrrrrrZtorch.fx.passesrZtorch.utils._ordered_setrZtorch.utils.checkpointrrZ-_activation_checkpointing.graph_info_providerr Z"_activation_checkpointing.knapsackr!r"r#Z,_activation_checkpointing.knapsack_evaluatorr$Z_aot_autograd.logging_utilsr%Z_aot_autograd.utilsr&r'Z compile_utilsr(r)r*ZsympyZdebug_partitionerr+rfrC getLoggerr?r,Loggerrrrr-rFrgrErpr^rur|rcrrrrrarrrrrrrrrrrrrrrrrrrrrrrrrrrr,r:rr<rFrKcacherPrbrTr`rertrrrrrrrrrNrr!r)r\rjr6rr8r8r8r9s                  G    N E  ;T    y   _  (N ^ L* ) .  * t /  O