a gh4O@sddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z ddl m Zddl mZddZd d Zd d Zd*ddZd+ddZd,ddZGdddZGdddZddZd-ddZd.d d!Zd"d#Zed/d&d'Zd0d(d)ZdS)1N)contextmanager)AnyDictList)language)runtimecCsLd|}dddd|dg}t|}|tjjd}dd|D}|S) N, nvidia-smi-i0z --query-gpu=z--format=csv,noheader,nounitscSsg|] }t|qS)int.0xr r znvsmi..)join subprocess check_outputdecodesysstdoutencodingsplit)attrscmdoutretr r rnvsmi s   r!cs0ttfddfdd|DS)Ncsbd|krdksntd|d}t|}t|}||}d||||S)Nrrz%Quantiles must be in the range [0, 1]) ValueErrormathfloorceil)qZpointloweruppert)anr r get_quantiles   z_quantile..get_quantilecsg|] }|qSr r )rr&)r,r rr'rz_quantile..)lensorted)r*r&r )r*r,r+r _quantiles r/cCs~|dur*t||}t|dkr&|d}|S|dkr6|S|dkrFt|S|dkrVt|S|dkrht|S|dkrzt|SdS)Nrrallminmaxmeanmedian)r/r-r1r2 statisticsr3r4)times quantiles return_moder r r r_summarize_statistics*s   r9r3c Csddl}|dvsJ|j|j||durZ|D]}||dd|_q<|jjdd}|jjdd}|t dD] } |q||j | |d} | dkrd} nt dt || } |j} |j| @t | D]&} |dur|D] }d|_q|qWdn1s00Y|j g} d }t |D]X} |jjdd}|jjdd}|| ||j | | || g7} qTt| ||WdS1s0YdS) a Benchmark the runtime of the provided function. :param fn: Function to benchmark :type fn: Callable :param rep: Repetition time (in ms) :type rep: int :param grad_to_none: Reset the gradient of the provided tensor to None :type grad_to_none: torch.tensor, optional :param return_mode: The statistical measure to return. Options are "min", "max", "mean", "median", or "all". Default is "mean". :type return_mode: str rNr1r2r3r4r0TZ enable_timingir )torchcudastreamZStreamZdetach_Zrequires_grad_gradEventrecordrange synchronize elapsed_timer2rZ CUDAGraphgraphZreplayr9)fnrep grad_to_noner7r8r?r start_event end_event_ estimate_msn_repeatgr Z n_retriesr r rdo_bench_cudagraph<sN         (   rRdcst|dvs Jtjj|tjj}jdd}jdd}|tdD]} tjj ||qZ|| |d} t dt || } t dt || } fddt| D}fddt| D}t| D] } |qt| D]L} |d ur|D] }d |_ qtjj ||| ||| qd dt||D}t|||S) a Benchmark the runtime of the provided function. By default, return the median runtime of :code:`fn` along with the 20-th and 80-th performance percentile. :param fn: Function to benchmark :type fn: Callable :param warmup: Warmup time (in ms) :type warmup: int :param rep: Repetition time (in ms) :type rep: int :param grad_to_none: Reset the gradient of the provided tensor to None :type grad_to_none: torch.tensor, optional :param quantiles: Performance percentile to return in addition to the median. :type quantiles: list[float], optional :param return_mode: The statistical measure to return. Options are "min", "max", "mean", "median", or "all". Default is "mean". :type return_mode: str r;Tr<r=rcsg|]}jddqSTr<rCriZdir rrrzdo_bench..csg|]}jddqSrUrVrWrYr rrrNcSsg|]\}}||qSr )rG)rser r rrr)rdriveractiveZget_device_interfacerFZget_empty_cache_for_benchmarkrCrDrE clear_cacherGr2rrBzipr9)rIZwarmuprJrKr7r8cacherLrMrNrOZn_warmuprPrXrr6r rYrdo_benchs>           rac CsNddl}ddl}t||js&||}t||js<||}|durHd}t|rZ||jn|}|durjd}t|r|||jn|}t||jr|j|jkr|}| }t||jr|j|jkr|}| }|j dks|j dkr|j j ||||dddS|j||||dsJt|d |d |d |d |d dS)a Asserts that two inputs are close within a certain tolerance. :param x: The first input. :type x: scala, list, numpy.ndarray, or torch.Tensor :param y: The second input. :type y: scala, list, numpy.ndarray, or torch.Tensor :param atol: The absolute tolerance. Default value is 1e-2. :type atol: float, optional :param rtol: The relative tolerance. Default value is 0. :type rtol: float, optional :param err_msg: The error message to use if the assertion fails. :type err_msg: str rNg{Gz?grT)atolrtolZ equal_nan)rcrd z is not close to z (atol=z, rtol=))numpyr? isinstanceZTensorZtensorcallabledtypebfloat16floatcpudetachsizeZtestingZassert_allcloseZallcloseAssertionError)ryrcrderr_msgnpr?r r r assert_closes2        rtc @sLeZdZdZdeeeeeeeeeeeeefeeeed ddZ dS) Benchmarkzk This class is used by the :code:`perf_report` function to generate line plots with a concise API. rbFN) x_namesx_valsline_arg line_vals line_names plot_nameargsxlabelylabelx_logy_logc CsL||_||_| |_||_||_||_| |_| |_||_| |_ ||_ ||_ dS)aq Constructor. x_vals can be a list of scalars or a list of tuples/lists. If x_vals is a list of scalars and there are multiple x_names, all arguments will have the same value. If x_vals is a list of tuples/lists, each element should have the same length as x_names. :param x_names: Name of the arguments that should appear on the x axis of the plot. :type x_names: List[str] :param x_vals: List of values to use for the arguments in :code:`x_names`. :type x_vals: List[Any] :param line_arg: Argument name for which different values correspond to different lines in the plot. :type line_arg: str :param line_vals: List of values to use for the arguments in :code:`line_arg`. :type line_vals: List[Any] :param line_names: Label names for the different lines. :type line_names: List[str] :param plot_name: Name of the plot. :type plot_name: str :param args: Dictionary of keyword arguments to remain fixed throughout the benchmark. :type args: Dict[str, Any] :param xlabel: Label for the x axis of the plot. :type xlabel: str, optional :param ylabel: Label for the y axis of the plot. :type ylabel: str, optional :param x_log: Whether the x axis should be log scale. :type x_log: bool, optional :param y_log: Whether the y axis should be log scale. :type y_log: bool, optional :param styles: A list of tuples, where each tuple contains two elements: a color and a linestyle. :type styles: list[tuple[str, str]] N) rvrwrrxryrzrstylesr}r~r{r|) selfrvrwrxryrzr{r|r}r~rrrr r r__init__s/zBenchmark.__init__)rbrbFFN) __name__ __module__ __qualname____doc__rstrrrboolrr r r rrus&  ruc@s4eZdZddZd eeeedddZd d d Zd S)MarkcCs||_||_dSN)rI benchmarks)rrIrr r rr9sz Mark.__init__F)bench save_path show_plots print_datac s|ddl}ddlm} ddl} |j} dd|jD} dd|jD} t|j}| j|| | | d}|jD] t tt fsfdd|Dt t |krt dt |dt t|}ggg}}}|jD]z}|jfi||j|i|j|}z|\} } } Wn$ty6|dd} } } Yn0|| g7}|| g7}|| g7}qt||||jt |<qj|jr| | }|d}t|jD]\}}||d ||d } } |jr|j|dnd}|jr|j|d nd}|j|||||||d | s| s| t} | t} |j||| | d |dq| |!|j"pv||#|j$|%|j&rdnd|'|j(rdnd|r| )|r| *|j+,||jd|||j}|r(|j-d dkr(|j./\}}|||||d<|rHt0|jdt0|1|rx|j2|j+,||jdd|ddd|S)NrcSsg|]}|dqS)-minr rr r rrDrzMark._run..cSsg|]}|dqS)-maxr rr r rrEr)columnscsg|]}qSr r )rrNrr rrKrz Expected z values, got rrr)labelcolorZlsg333333?)alpharlogZlinearz.pngZDiff:z.csvz%.fF)Z float_formatindex)3osZmatplotlib.pyplotZpyplotZpandasrzlistrvZ DataFramerwrhtupler-r"dictr_ryrIrxr| TypeErrorlocr{figureZsubplot enumeraterZplotZisnullr0ZastyperlZ fill_betweenZlegendZ set_xlabelr}Z set_ylabelr~Z set_xscalerZ set_yscalershowZsavefigpathrshapertolistprintZ to_stringZto_csv)rrrrrZdiff_colZsave_precisionZkwragsrZpltpdZy_meanZy_minZy_maxrvZdfZx_argsZrow_meanZrow_minZrow_maxrqr axZfirst_xrXcolZstyZcol0Zcol1r rr_run=sv    $        "z Mark._runrbc Ks|t|jt}|r|jgn|j}g}z|D]"} ||j| |||fi|q(W|rtj|ddttj |ddN} | d|dt |D]} | d| j dq| dWdn1s0Yn|r^tj|ddttj |ddP} | d|dt |D]} | d| j dq| dWdn1sT0Y0|rx|rt|d S|SdS) NT)exist_okz results.htmlwz z z r) rhrruappendrrmakedirsopenrrwriter-r{) rrrrZ return_dfkwargsZhas_single_benchrZ result_dfsrhtmlr r rruns2 " * ,zMark.runN)Fr)FFrbF) rrrrrurrrrr r r rr7s Ercsfdd}|S)z Mark a function for benchmarking. The benchmark can then be executed by using the :code:`.run` method on the return value. :param benchmarks: Benchmarking configurations. :type benchmarks: List of :class:`Benchmark` cs t|Sr)r)rIrr rrzperf_report..r )rwrapperr rr perf_reports rcCs^ddl}ddlm}|s"|j}|jj|d}|jj|d}||ddd }|S) z return DRAM bandwidth in GB/s rNrr\Zmem_clock_rateZ mem_bus_widthrg.A)r?rr\r@current_devicer]utilsget_device_properties)devicer?r\Z mem_clock_khzZ bus_widthZbw_gbpsr r r get_dram_gbpss  rc Csddl}ddlm}|s"|j}|jj|dd}|j|}|ddkrd||j ks^Jd}nV||j |j fvrzd}n@||j |j |j fvrd}n&||jtjtjtjfvrd }ntd |||d }|S) Nrrrmultiprocessor_countriidtype not supported& .>)r?rr\r@rr]rrget_device_capabilityfloat16float32int32rkint16Zint8tlZ float8e4nvZ float8e4b15Zfloat8e5 RuntimeError rjZ clock_raterr?r\Z num_subcoresZ capabilityZops_per_sub_coretflopsr r rget_max_tensorcore_tflopss$    rc sfdd}|S)Ncstfdd}|S)Nc sddl}|t}|k}|r|dkrtjjd}tj ddd}d|vshJd|dj j j }|d j d |d }tjdd d |gd|d} | jdksJddt| jvsJn|i|dS)Nrz cuda-memcheck__file__PATH1)rZPYTORCH_NO_CUDA_MEMORY_CACHINGrequestz@memcheck'ed test must have a (possibly unused) `request` fixturez::[]Zpytestz-vsT)capture_outputenvz7cuda-memcheck returned an error: bounds checking failedzERROR SUMMARY: 0 errors)psutilProcessrgetppidnameitemsrrealpath __globals__environnodeZcallspecidrrr returncoderr) r|rrZ ppid_nameZrun_cuda_memcheckrrZtest_idrr) target_kwargstest_fnr rrs z1cuda_memcheck..decorator..wrapper) functoolswraps)rrr)rr decoratorsz cuda_memcheck..decoratorr )rrr rr cuda_memchecks rFc cs$ztgdtdddd|d|gtdddd|d|gtdgd }td gd }t||d ksJd |d t||d ksJd |d d|}d|d}||fVWtgdtgdtgdn,tgdtgdtgd0dS)N)r r r -pmrr r r z--lock-gpu-clocks=r z--lock-memory-clocks=zclocks.current.smrzclocks.current.memoryr>zGPU SMs must run at z MHzg3O?igMbP?)r r r rr )r r r z-rgc)r r r z-rmc)rrr!abs)Z ref_sm_clockZ ref_mem_clockZ cur_sm_clockZ cur_mem_clockrZgbpsr r r set_gpu_clocks6    rc Csddl}ddlm}|s"|j}|jj|dd}|j}|ddkrx||j kr^d}q||j krnd}qt d n.||j krd}n||j |j fvrd}nt d |||d }|S) Nrrrrrr @rr) r?rr\r@rr]rrrrrrrkrr r rget_max_simd_tflopss&        r)r:NNr3)rSrTNNr3)NNrb)N)N)rr)N)rr#rr5rr contextlibrtypingrrrrbrrrr!r/r9rRrartrurrrrrrrr r r rs0     C B 3Cc