a hgp$@sUddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z ddl mZddlmZddlmZdd lmZdd lmZdd lmZddlZd d gZe dZedZejjZddZiZ e!eefe"d<ddZ#dRe e eefge eeffddd Z$e$ej%dde&dddZ'e$ej(dSe&dddZ)e$ej*dTe&dddZ+e$ej,dUe&ddd Z-e$ej.dVe&dd!d"Z/dWe0e&e0e&e0e&e1e&d#d$d%Z2e$ej3ej4gdde&dd&d'Z5e$ej6e&dd(d)Z7d*d+Z8e$ej9ej:ej;gdde&dd,d-Ze>e&d1fe>e&d1fe>e&d1fe e>e&d1ffdd2d3Z?dd0ee>e>e&d1fe>e&d1fe>e&d1fe e>e&d1ffdd4d5Z@e$ejAd6d7dde&dd8d9ZBe$ejCd6d7e&dd:d;ZDdd?ZIe$ejJd6d7e&dd@dAZKe$ejLd6d7e&ddBdCZMej%e'ej(e)ej*e+ej,e-ej.e/ej3e5ej4e5ej6e7ej9e.nfrr&r'rr%r shape_wrappersr)Freturncs*tttftttfdfdd }|S)N) flop_formular+cs,s tfdd}tjj|S)NcsHt|tjjs&td|dt||tvr.register_fun..register)r)rutilsZ_pytreeZ tree_map_)r,r2get_rawtargetsr1r register_fun$s  z+register_flop_formula..register_fun)r rr)r6r5r7rr4rr#s&)r"c Os,|\}}|\}}||ksJ||d|S)zCount flops for matmul.r) a_shapeb_shaper"r#r$mkk2nrrrmm_flop9s r?cKs t||S)zCount flops for addmm.r?Z self_shaper9r:r"r$rrr addmm_flopDsrBc KsD|\}}}|\}}} ||ks J||ks,J||| d|} | S)z"Count flops for the bmm operation.r8r) r9r:r"r$br;r<b2r=r>floprrrbmm_flopIs     rFcKs t||S)z&Count flops for the baddbmm operation.rFrArrr baddbmm_flopVsrHc Ks t||S)zCount flops for _scaled_mm.r@) r9r:Z scale_a_shapeZ scale_b_shapeZ bias_shapeZscale_result_shapeZ out_dtypeZuse_fast_accumr"r$rrr_scaled_mm_flop]srI)x_shapew_shaper" transposedr+c CsJ|d}|r|n|dd}|^}}}t|t||||d} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr8Nr) rJrKr"rLZ batch_sizeZ conv_shapeZc_outZc_inZ filter_sizerErrrconv_flop_countns  rMc Ost||||dS)zCount flops for convolution.rL)rM) rJrK_bias_stride_padding _dilationrLr"r#r$rrr conv_flopsrSc Csdd} d} | dr4t| d}| t|||| 7} | drt| d}|rn| t| || || |dd7} n | t| || || |dd7} | S)NcSs |d|dgt|ddS)Nrrr8)list)rrrrtszconv_backward_flop..trrFrN)rrM)grad_out_shaperJrKrOrPrQrRrLZ_output_paddingZ_groupsZ output_maskr"rU flop_countZgrad_input_shapeZgrad_weight_shaperrrconv_backward_flopsH  " rXcCs|\}}}}|\}}} } |\} } } }||kr8| krlnn0||krP| krlnn|| krl| | krl|| kspJd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rrG) query_shape key_shape value_shaperChs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   L""rhcOs t|||S)Count flops for self-attention.rh)rYrZr[r"r#r$rrr sdpa_flopsrkcCsRddlm}ddlm}t|||fs>|jjdkr>|S|g| ddS)z If the offsets tensor is fake, then we don't know the actual lengths. In that case, we can just assume the worst case; each batch has max length. r) FakeTensor)FunctionalTensormetar) Ztorch._subclasses.fake_tensorrlZ#torch._subclasses.functional_tensorrmrdevicer.difftolistsize)offsetsmax_lenrlrmrrr_offsets_to_lengthss    ru)grad_out.ccs&|durt|jdksJt|jdks,J|dusD|j|jksDJ|j\}} } |j\}} } |j\}} }|dustJ|dusJ|j|jksJt||}t||}t||D]J\}}d| || f}d| || f}d| ||f}|dur|nd}||||fVqdS|j|j|j|dur|jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrruzip)querykeyvaluerv cum_seq_q cum_seq_kmax_qmax_k_h_qr^h_kd_kh_vrfZ seq_q_lengthsZ seq_k_lengthsZ seq_q_lenZ seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shaperrr%_unpack_flash_attention_nested_shapes*s(          rccs.|durt|jdksJt|jdks.J|dusF|j|jksFJ|j\}}} } |j\}}} } |j\}}} }|dus|J|dusJ|j|jksJt||}t||}t||D]J\}}d| || f}d| || f}d| ||f}|dur|nd}||||fVqdS|j|j|j|dur"|jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrrx)r{r|r}rv cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krrr^rrrrfZ seqlens_qZ seqlens_klen_qZlen_krrrrrrr)_unpack_efficient_attention_nested_shapesXs(        rT)r5c Os(t|||||||d} tdd| DS)ri)r{r|r}r~rrrcss"|]\}}}}t|||VqdSrrj.0rYrZr[rrrr s z0_flash_attention_forward_flop..rsum) r{r|r}r~rrrr"r#r$sizesrrr_flash_attention_forward_flops rc Os(t|||||||d} tdd| DS)ri)r{r|r}rrrrcss"|]\}}}}t|||VqdSrrjrrrrrs z4_efficient_attention_forward_flop..rr) r{r|r}biasrrrrr#r$rrrr!_efficient_attention_forward_flops rcCsRd}|\}}}}|\} } } } |\} }}}|\}}}}|| krR| krR|krnn*|| krt|krt|krnn|| ksJ||kr| |kr||ksJd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)NrrG)rVrYrZr[rgrCr\r]r^r_r`rarbrcrdrerfZ_b4Z_h4Z_s4Z_d4rrrsdpa_backward_flop_counts    P"""""rcOst||||S)z(Count flops for self-attention backward.r)rVrYrZr[r"r#r$rrrsdpa_backward_flopsrc Os*t|||||||| d} tdd| DS)N)r{r|r}rvr~rrrcss$|]\}}}}t||||VqdSrrrrYrZr[rVrrrrs z1_flash_attention_backward_flop..r) rvr{r|r}outZ logsumexpr~rrrr#r$shapesrrr_flash_attention_backward_flops rc Os*t|||||||| d} tdd| DS)N)r{r|r}rvrrrrcss$|]\}}}}t||||VqdSrrrrrrr&s z5_efficient_attention_backward_flop..r) rvr{r|r}rrrrrrr#r$rrrr"_efficient_attention_backward_flop s rcCst|ts|fS|Sr)rtuple)xrrrnormalize_tupleAs r)KMBTcCs0tdtttdtt|dd}t|S)Nrrr8rw)maxminrysuffixesstr)numberindexrrrget_suffix_strJs(rcCs&t|}|d|d}|t|S)Niz.3f)rr)rsuffixrr}rrrconvert_num_with_suffixQs rcCs|dkr dS||dS)Nr0%z.2%r)numZdenomrrrconvert_to_percent_strXsrcstfdd}|S)Ncst|\}}|}t||Sr)rr)r#Z flat_argsspecrr%rrr'^s z)_pytreeify_preserve_structure..nfrr(rr%r_pytreeify_preserve_structure]srcseZdZdZdeeejje ejjfe e ee e e fdfdd Ze dd d Ze ee e e ffdd d Zdd dZddZddZddZZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() Nr8T)modsdepthdisplaycustom_mappingcstttdd|_||_||_d|_|dur6i}|durLtjdddit dd| D|_ t |_ dS)NcSsttSr)rintrrrrz*FlopCounterMode.__init__..zrz,FlopCounterMode.__init__..) super__init__r flop_countsrrmodewarningswarnr itemsr mod_tracker)selfrrrr __class__rrr{s zFlopCounterMode.__init__r*cCst|jdS)NGlobal)rrvaluesrrrrget_total_flopsszFlopCounterMode.get_total_flopscCsdd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. cSsi|]\}}|t|qSr)dictrrrrrrz3FlopCounterMode.get_flop_counts..)rrrrrrget_flop_countss zFlopCounterMode.get_flop_countsc s|durj}|durd}ddl}d|_gd}g}tdfdd}tjD]>}|dkrvqh|d d }||krqh|||d }| |qhdjvrވs|D]} d | d| d<q|dd|}t |dkrgd g}|j||d dS)Ni?BrT)ModuleZFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]0\}}||dt|t|t|gqX|S)N z - )rrrappendrrrr)mod_namerrgpaddingrr<rZ global_flopsZ global_suffixZis_global_subsumedrrr process_mods  z.FlopCounterMode.get_table..process_modr.rr)r0r)leftrightr)headersZcolalign) rtabulateZPRESERVE_WHITESPACErrsortedrkeyscountextendry) rrrheaderrrmodZ mod_depthZ cur_valuesr}rrr get_tables6   zFlopCounterMode.get_tablecCs,|j|jt||_|j|Sr)rclearr __enter___FlopCounterModerrrrrrs     zFlopCounterMode.__enter__cGsD|jdusJ|jj|}d|_|j|jr@t||j|Sr)r__exit__rrprintrr)rr#rCrrrrs  zFlopCounterMode.__exit__cCsV||jvrR|j|}||i|d|i}t|jjD]}|j|||7<q6|S)Nr!)r setrparentsr)rZ func_packetrr#r$Zflop_count_funcrWparrrr _count_flopss   zFlopCounterMode._count_flops)Nr8TN)N)__name__ __module__ __qualname____doc__rr rnnrrTrboolrrrrrrrrrr __classcell__rrrrrgs" = c@s$eZdZedddZdddZdS) rcountercCs ||_dSrr)rrrrrrsz_FlopCounterMode.__init__rNcCs6|r|ni}|tjjjjtjjjjtjjjjtjjjjtjjjjtjjj jtjjj jtjjj jtjjj jtjjj jtjjjjtjjjjtjjjjtjjjjhvrtS||jjvr|tjjjjur|:|j|i|}|tur|WdSWdn1s 0Y||i|}|j|j|||Sr)ropsatenZ is_contiguousdefaultZ memory_formatZis_strides_like_formatZis_non_overlapping_and_denserrZsym_sizeZstrideZ sym_strideZstorage_offsetZsym_storage_offsetZnumelZ sym_numelZdimZprimZlayoutNotImplementedrr roZ decomposerZ_overloadpacket)rfunctypesr#r$rrrrr__torch_dispatch__s0                2z#_FlopCounterMode.__torch_dispatch__)rN)rrrrrrrrrrrsr)F)N)N)N)NNNFN)F)VrZtorch.utils._pytreerrrZmodule_trackerrtypingrrr r r collections.abcr Ztyping_extensionsr collectionsrZtorch.utils._python_dispatchrmathr functoolsrr__all__rrrrrr r__annotations__r)rmmrr?ZaddmmrBZbmmrFZbaddbmmrHZ _scaled_mmrIrTrrMZ convolutionZ _convolutionrSZconvolution_backwardrXrhZ'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionZ#_scaled_dot_product_cudnn_attentionrkrurrrZ_flash_attention_forwardrZ_efficient_attention_forwardrrZ0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardZ,_scaled_dot_product_cudnn_attention_backwardrZ_flash_attention_backwardrZ_efficient_attention_backwardrrrrrrrrrrrrrs        *    ' g 6 3 6 0     !