a hv@sBdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZmZmZmZddlZddlmZddlZddlmZddlmZddlm Z dd l!m"Z"dd l#m$Z$m%Z%d d l&m'Z'd d lm(Z(m)Z)e*e+Z,edZ-edZ.e./Z0e0rLddl1Z2gZ3dZ4e0rgdZ3e2j5j6j789ddZ:d;dde3DZ4gdZdZ?GdddZ@ejAdd ZBd!d"d#d$ZCd!d"d%d&ZDd'd(ZEd)d*ZFGd+d,d,eGZHd-d.ZIdRd/d0ZJdSd!d!d1d2d3ZKd4d5ZLd6d7ZMd8d9ZNdTd!d!d1d:d;ZOed<d=dd?d@ZPe-eee-ge-fdAdBdCZQeQejRZSeQeTdDZUeQdZVeQd!ZWeQd!ZXGdEdFdFZYGdGdHdHZZGdIdJdJZ[dUee\ege\efe]ee^e]e_fee_e^e]efdLdMdNZ`e]ee-ge-fdOdPdQZadS)Va Debug utilities for TorchDynamo compilation and execution. This module provides various debugging tools and utilities for TorchDynamo, including: - Minification support for reducing test cases while preserving bugs - Input/output handling via InputReader and InputWriter for reproducible testing - Accuracy checking between original and compiled models - Neural network module string conversion via NNModuleToString - Profiling tools and system information collection - Buck build system integration for Meta-internal testing Key classes: - InputReader/InputWriter: Handle serialization of model inputs/outputs - NNModuleToString: Converts nn.Modules to string representations - BuckTargetWriter: Manages Buck build system integration N)Counter) import_module)AnyCallableOptionalTypeVar)Tensor) rand_strided)is_float_dtype)StorageWeakRef)ContentStoreReaderContentStoreWriter)config) clone_inputs get_debug_dirTztorch._inductor.config)z1//caffe2/torch/fb/sparsenn:sparsenn_operators_gpuz-//caffe2/torch/fb/sparsenn:sparsenn_operatorsz///deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpuz+//deeplearning/fbgemm/fbgemm_gpu:sparse_opszfbcode:// cCsg|]}d|dqS)ztorch.ops.load_library("z").0xrrG/var/www/auris/lib/python3.9/site-packages/torch/_dynamo/debug_utils.py Kr)Zbuck2runz@mode/dev-nosanc@s&eZdZddZddZd ddZdS) BuckTargetWritercCstjtj|\|_|_|jdd|_|jddd|j|_|j|jdd|_|jdd|_|j}||dddd}d|d |j|_ dS) Nz.pyr/.zfbcode.zfbcode/r:) ospathsplitabspathsubdirpy_filereplacetargetfind cmd_line_path)selffilenametmprrr__init__RszBuckTargetWriter.__init__c CsDdddtD}td|jd|jdtd|d|jd S) NrcSsg|]}d|dqS)z "z",rrrrrrarz*BuckTargetWriter.build..za load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") python_binary( name="z", srcs = ["z"], compile = False, deps = [ "//caffe2:torch", "//caffe2:libtorch", "//caffe2/functorch:functorch", "//triton:triton", "z", ], cpp_deps = [ z ], main_module = "z", par_style = "xar", ) )join extra_depstextwrapdedentr*r( cur_targetr$)r-Zextra_cpp_depsrrrbuild`s zBuckTargetWriter.buildTcCsntj|jd}t|d}||Wdn1s>0Yt|jg}|rjt dd||S)NZTARGETSwzFFound an example that reproduces the error. Run this cmd to repro - %s ) r#r$r1r'openwriter6BUCK_CMD_PREFIXr,logwarning)r-Z print_msgZ target_filefdZ cmd_splitrrrr:zs , zBuckTargetWriter.writeN)T)__name__ __module__ __qualname__r0r6r:rrrrrQsrcCsLtjtd}|dur.tdt}tj|sHtj |dd|S)NZminifierz /minifier_T)exist_ok) r#r$r1rtempfile gettempdirgetpassgetuserexistsmakedirs)r$rrr minifier_dirs  rIc@seZdZejjejjejjejjejj ejj ejj ejj ejj ejjejjejjejjejjejjejjejjejjejjejjejjgZeddZeddZdS)NNModuleToStringcCsLt}|D] \}}t|tjvr||qt|dkrHtd|dS)Nrz-We have not tested reprs of some modules - %sT) setnamed_childrentyperK safe_reprsaddlenr<r=)gmZ cant_convert_modulerrrcan_convert_to_strings   z&NNModuleToString.can_convert_to_stringcCsddlm}d}td}|D]V\}}|}t|d}|durZ|jrZ|d}||dd|d|d 7}q"|j D]\}} | durq| t krdd l m} | jt ksJt| } n@t| rd t| jd | jd } ndt| jd | jd } | jr| d} ||dd|d| d7}q|j D]d\} } | durXqBd}| jrhd}dt| jd | j|d} ||dd| d| d 7}qB|||jdd 7}|S)Nr) _addindent z from torch.nn import * class Repro(torch.nn.Module): def __init__(self) -> None: super().__init__() z.cuda()zself.z = r) PRINT_OPTSz torch.randn(z, dtype=)ztorch.randint(1, size=zself.register_buffer('z', z) rz, device="cuda"ztorch.nn.Parameter(torch.randn(z))rJ)Ztorch.nn.modules.modulerVr3r4rM__repr__next parametersZis_cuda_buffersitemsZnumelMAX_CONSTANT_NUMEL_INLINEZtorch._tensor_strrY thresholdreprtorchis_floating_pointlistshapedtype _parameterscode)rRrVtab model_str module_namerTZ module_strZ example_paramZ buffer_namebufferrYZ tensor_str param_nameparam maybe_devicerrrconvertsJ           " zNNModuleToString.convertN)r?r@rArcnnZLinearZConv1dZConv2dZConv3dZ BatchNorm1dZ BatchNorm2dZ BatchNorm3dZ LayerNormZDropoutZSoftmaxZReLUZGELUZIdentityZ MaxPool2dZ EmbeddingZTanhZConvTranspose1dZGLUZLSTMZFlattenZAdaptiveAvgPool2drO staticmethodrUrqrrrrrKs4 rKc CstjsdSd}zBtddg}|d}ddd|D}||d7}Wn ttj fyt|d 7}Yn0t d d t tj D}|d 7}| D]\}}|d |d|d7}q|d7}|S)Nz:# torch.cuda.is_available()==False, no GPU info collected z# CUDA Info: Znvccz --versionrrcSs g|]}|dvrd|dqS))r#  r)rsrrrrrz-_cuda_system_info_comment..z# nvcc not found css|]}tj|VqdSN)rccudaZget_device_name)rirrr sz,_cuda_system_info_comment..z# GPU Hardware Info: rtz : ru)rcrxZ is_available subprocess check_outputdecoder%r1FileNotFoundErrorCalledProcessErrorrrangeZ device_countr_)rkZcuda_version_outZcuda_version_linescommentZ gpu_namesnamecountrrr_cuda_system_info_comments$  rF) stable_outputcsT|rdSgdgdfddfddtjD}d|}d |d S) zl Generate a string configuration for environment variables related to Dynamo, Inductor, and Triton. z+# env var omitted due to stable_output=True)ZTORCHZDYNAMOZINDUCTORZTRITON)ZTRITON_LIBDEVICE_PATHZTRITON_PTXAS_PATHZTRITON_LIBCUDA_PATHcstfddDovS)Nc3s|]}|vVqdSrwr)rstringkeyrrrzrz;generate_env_vars_string..filter..)anyr) allow_list skip_listrrfiltersz(generate_env_vars_string..filtercs*g|]"\}}|rd|d|dqS)z os.environ['z'] = ''rrrvalue)rrrr sz,generate_env_vars_string..rz import os z )r#environr_r1)rZ config_linesZ config_stringr)rrrrgenerate_env_vars_strings  rc Cs\ddl}ddl}|rdS|jjj}d|jjd|jjd|j jd|d S)Nrz*# config omitted due to stable_output=Truez~import torch._dynamo.config import torch._inductor.config import torch._functorch.config import torch.fx.experimental._config r) Ztorch._functorch.configZtorch._inductor.configZfxZ experimental_configZcodegen_config_dynamorZ _inductorZ _functorch)rrcZexperimental_configrrrgenerate_config_string,s   rcCstjtdS)Nzminifier_launcher.py)r#r$r1rIrrrrget_minifier_repro_path@src Cst}td|tr"t|z8t|d}||Wdn1sN0YWn8ty}z tdt d|WYd}~n d}~00dS)NzWriting minified repro to: %sr7rz(Could not write to {minified_repro_path}) rr<r=use_buckrr:r9OSError exceptionNotImplementedError)contentsZminified_repro_pathr>errrhelper_for_dump_minifyDs   , rc@s eZdZdS) AccuracyErrorN)r?r@rArrrrrSsrcCsBt|}tt|D](}t||tjr||||jq|S)z This clone inputs is different from utils clone_input. In case of minifier, all the tensors are leaf tensors while creating a new graph. So, we set the requires_grad field w/o checking the leafness of the tensor. )rrrQ isinstancercrZrequires_grad_ requires_grad)example_inputsZ cloned_inputsidxrrrclone_inputs_retaining_gradnessWs rc Csddlm}m}m}t|}|s*t|}t|dr>|dt |ddrR||n||}|rb|S||rz||}| |||d|S)z Runs a forward and possibly backward iteration for a given mod and args. When disable_clone is True, we will use args as-is without cloning. This is higher fidelity but we may destroy the args in the process. r)collect_resultsreduce_to_scalar_lossrequires_bwd_pass zero_gradTZ _boxed_callFN) ZtestingrrrcopydeepcopyrhasattrrgetattrZbackward) rRargsonly_fwdZ disable_clonerrroutZlossrrrrun_fwd_maybe_bwdds   r require_fp64 ignore_non_fpc Csddlm}t|||}d}tjrtz(tt|t|\} } t| | |}Wn(t yr|rdt dt dYn0zt|||} Wnt yt dYdS0||| |tjd|d} | S) aa Check two models have same accuracy. require_fp64: if True, raise an error if we unable to calculate the fp64 reference ignore_non_fp: if True, do not compare outputs which are not floating point. This is mostly useful for the minifier (which wants to avoid quantizing floating point error into integer/boolean error) r)sameNzfCould not generate fp64 outputs, workaround with torch._dynamo.config.same_two_models_use_fp64 = FalsezCould not generate fp64 outputszWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graph.T)ZtolZ equal_nanr)utilsrrrZsame_two_models_use_fp64 cast_to_fp64rrr Exception RuntimeErrorr<r=rZrepro_tolerance) rRZopt_gmrrrrrrefZfp64_refZ fp64_modelZ fp64_examplesresZpassingrrrsame_two_modelss>    rcCs|jjD]}|jdkrj|jtjjjjkrjt |j dks:Jt |j drj|j dtj krj|j dtj f|_ |jdkr|j d}|durt |rt|j }tj |d<||_ q|j||S)N call_functionrXrrrg)graphnodesopr*rcopsZprimsZconvert_element_typedefaultrQrr float64kwargsgetdictZlintZ recompile)modelnodergZ new_kwargsrrrcast_dtype_args_to_fp64s"      rcsBddlm}|}tjkr(t|}|fdd|}||fS)Nr)tree_mapcs"t|tjr|r|S|Srw)rrcrrdtorrgrrs   zcast_to..)Ztorch.utils._pytreerrrcrr)rgrinputsrrrrcast_tos    rcCsttj||Srw)rrcr)rrrrrrsrcCsPz,|t|t|}t||||||d WStyJtdYdS0dS)NrzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graphF)rrrrrr<r)rRrZ compiler_fnrrrZ compiled_gmrrrbackend_accuracy_failss"   rztorch._prims_common.StrideTypeztorch._prims_common.ShapeType)striderfreturncCs|dur |St|Srw)rZmake_contiguous_strides_for)rrfrrr_stride_or_default sr)drcs fddS)Ncs|dur |SSrwrrrrrrrz_mk_defaulter..rrrrr _mk_defaultersrcpuc@s:eZdZddddZdddddZdd Zd d ZdS) NopInputReaderNrcCs d|_dS)Nrtotal)r-rrrr0!szNopInputReader.__init__device dtype_hintcCs|jd7_dS)Nrr)r- storage_hashnbytesrrrrrstorage$szNopInputReader.storagecOsdSrwrr-rrrrrtensor'szNopInputReader.tensorcOsdSrwrrrrrsymint*szNopInputReader.symintr?r@rAr0rrrrrrrr src@sJeZdZd ddddZdddddZddddddd d Zd d ZdS) InputReaderN)pbarcCs8|durtd|dur"t|nd|_g|_||_dS)Nz0no save_dir specified, will generate random data)r<r=r storerr)r-save_dirrrrrr01s  zInputReader.__init__rcCs|jdur|jdt|}t|}|jdurz|durzz|j|}WntyZYn 0||jkrvt d||j|St d|||j f}t d|d}t |||| S)Nrzdevice mismatch: %s != %sz1could not load %s, generating random data insteadrf)rupdate_device_or_default_dtype_or_defaultrZ read_storager~rr<r=itemsizerr untyped_storage)r-rrrrrrfrrrrr<s        zInputReader.storage)storage_offsetrgris_leafc Ks t||d}t|}t|}t|}t|}tjg||j|d} t | ||||Wdn1sn0Y|st | j tj d} Wdn1s0Yt | ||||Wdn1s0Ytj j| |ksJtj| ||j| | S)Nr)rgrr)Z memory_format)r_storage_offset_or_defaultr_is_leaf_or_default_requires_grad_or_defaultrcrrZno_gradset_Z enable_gradcloneZpreserve_format _subclasses meta_utils safe_is_leaf_utilsZset_tensor_metadatarappend) r-rrfrrrgrrmetadatatrrrrRs&   . , . zInputReader.tensorcCs|j||Srw)rr)r-valrrrrss zInputReader.symint)N)Nrrrrrr0s  !rc@sjeZdZddddZddZddded d d Zdd d d ZddZdd ddZ dd ddZ dS) InputWriterF stable_hashcCs:g|_t|_||_|dur*t||dnd|_i|_dS)Nr)_lines itertoolsrstorage_counterrr r seen_storages)r-rrrrrr0s zInputWriter.__init__cCs*dg}|dd|jD|d|S)Nzdef load_args(reader):css|]}d|VqdS)rWNr)rlrrrrzrz$InputWriter.lines..zload_args._version = 0)extendrr)r-rrrrliness  zInputWriter.linesNr device_hintrc Cst|}|j|}|dur |Sdt|j}d}tdt|krNd|}d}|j}|jdkrr|dusnJ|}td|krd|}| } d} |j dur|jjdkr|j |} |j |d| d| ||d||j|<|S) Nbufrz , dtype_hint=metaz , device=z = reader.storage(, rZ)r rrr\rrrrNrrrZ write_storagerr) r-rrr wsvZmaybe_dtype_hintrprrrrrrrs0        zInputWriter.storagec CsPddlm}m}|j||j|jd}g}||td|jd| s^| t t | t d|jkr~| d|j|td|ks| d|tj|}|r|dd|Dtd|jkr| d |jtjj|}td|kr| d ||j d d |t t |jg|d |dS)Nr)statically_known_truesym_eqrrzdtype=zstorage_offset=css |]\}}|d|VqdS)=Nr)rkrrrrrzrz%InputWriter.tensor..zrequires_grad=zis_leaf=zreader.tensor(r ) # )Z%torch.fx.experimental.symbolic_shapesrrrrrgrrrfrrstrtuplerrrrcrZget_tensor_metadatarr_rrrrrrrr1) r-rrrrrrZtensor_metadatarrrrrs< zInputWriter.tensorcCs|jd|dt|t|ttfr|jdt|D]\\}}|d|d}t|tjrp| ||q>t|t tj fr| ||q>| ||q>|jddS)Nrtz# was unsupported type for dumping: z"""[])rrrNrrer enumeratercrrintSymIntr unsupported)r-rargryaZname_irrrrs  zInputWriter.unsupportedcCs|jd|d|ddS)Nz reader.const(rz!, filtered out during compilation)rr)r-rrrrconstszInputWriter.constcCs0t|tjr|jj}|jd|d|dS)Nzreader.symint(r)rrcrrhintrr)r-rrrrrrs zInputWriter.symint) r?r@rAr0rrrrrrrrrrrrs  rrx)funcr sym_shapesdefault_sym_shapercsddlm}dd|D}d|}t|}d|d}d|d } d } Gd d d } i} pjifd dtdfdd } |j}|D]t\}}|dkrqt | |}|r| \}}t | d}||}| ||| |<t | |}|r|d| |<qdt|jvr~| }|| d<t ||D]>}| \}}}}t | d}||}t||| ||q>| S)a Takes in a function which has been printed with print_readable() and constructs kwargs to run it. Handles Tensor inputs, Symints, and a graph module which might have tensor constants. Consider a function `forward` defined as follows: def forward(self, primals_1: "f32[1001, 6]", primals_2: "f32[s0]", primals_3: "Sym(s0)",): _tensor_constant0: "i64[4190]" = self._tensor_constant0 # Further implementation kwargs = aot_graph_input_parser(forward) forward(**kwargs) r) dtype_abbrscSsi|]\}}||qSrrrrrr rz*aot_graph_input_parser..|z(_tensor_constant\d+): \"(z0)\[\s*(.*?)\s*\]\" = self\.(_tensor_constant\d+)(z)\[\s*(.*?)\s*\]z Sym\((s\d+)\)c@seZdZdZdS)z/aot_graph_input_parser..TensorContainerz#Container for tensors as attributesN)r?r@rA__doc__rrrrTensorContainersr(cs,tvpdufddS)Ncs dS)Nz; not in symbolic_shapes and default sym shape not passed inrrrrrr'rz=aot_graph_input_parser..get_sym_int..)rc_checkrr))r"r!r)r get_sym_int$s  z+aot_graph_input_parser..get_sym_intrc sg}g}t|D]H\}}|}d|vrF|}||||q|r|t|q|jrftjntj}|||d}|D]} tj || q~|S)Nrv)rgr) rstriprrrdrcZrandnzerosrZ mark_dynamic) rfrgZresolved_shapeZ dynamic_dimsryZdimrv constructorrr)rr+rr gen_tensor+s  z*aot_graph_input_parser..gen_tensorr,rr-)Ztorch.utils._dtype_abbrsr#r_r1valuesinspect getsourcer__annotations__researchgroupsrr%group signaturer]finditersetattr)r rr!r"r#Z dtype_mapZ dtype_patternsourceZtensor_assignment_regexZ tensor_regexZsym_shape_regexr(rr/ annotationsro annotationmatchZ data_typeZ shape_strrfrg container attr_namerSr)r"rr+r!raot_graph_input_parsersD       rB)r.rcsDttjtjfdd}fdd}t||S)z Decorator to cProfile a given function and save the result to disk on process exit. Args: filename: filename to save profile to cstfdd}|S)Ncs0z|i|WS0dSrw)enabledisable)rr)fnprofrrwrapperhs z3profile_to_file..decorator..wrapper) functoolswraps)rErG)rF)rEr decoratorgsz"profile_to_file..decoratorc s.tjtddddS)Nz! Wrote profile to z+, view with: snakeviz z )Z dump_statssysstderrr:r3r4rr.rFrrsave_itrs z profile_to_file..save_it)cProfileZProfiler#r$r& expanduseratexitregister)r.rJrNrrMrprofile_to_file]s   rS)FF)F)F)rxNN)br'rQrrOrHrEr2rloggingr#r5r{rKrCr3 collectionsr importlibrtypingrrrrrcZtorch._prims_commonZ _prims_commonrZtorch._subclasses.meta_utilsrZtorch._dynamo.testingr r Z torch.multiprocessing.reductionsr Ztorch.utils._content_storer r rrrr getLoggerr?r<rZinductor_configZ is_fbcoderZlibfb.py.build_infoZlibfbr2Z extra_importspyZ build_infoZ BuildInfoZget_build_ruler)r5r1r;rrIr`rKcacherrrrrrrrrrrrrrrrfloat32rrrrrrrrrrerrrrBrSrrrrs         7 f   : *  Tx c