o wZhgp @sTUddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z ddl mZddlmZddlmZdd lmZdd lmZdd lmZddlZd d gZe dZedZejjZddZiZ e!eefe"d<ddZ#dUde e eefge eefffdd Z$e$ej%ddde&fddZ'e$ej(dVde&fddZ)e$ej*dVde&fddZ+e$ej,dVde&fdd Z-e$ej.     dWde&fd!d"Z/ dUd#e0e&d$e0e&d%e0e&d&e1de&f d'd(Z2e$ej3ej4gddde&fd)d*Z5e$ej6de&fd+d,Z7d-d.Z8e$ej9ej:ej;gddde&fd/d0Ze>e&d4fe>e&d4fe>e&d4fe e>e&d4fffd5d6Z?dd3dee>e>e&d4fe>e&d4fe>e&d4fe e>e&d4fffd7d8Z@e$ejAd9d:ddde&fd;d<ZBe$ejCd9d:de&fd=d>ZDd?d@ZEe$ejFejGejHgddde&fdAdBZIe$ejJd9d:de&fdCdDZKe$ejLd9d:de&fdEdFZMiej%e'ej(e)ej*e+ej,e-ej.e/ej3e5ej4e5ej6e7ej9e<ej:e<ej;e<ejFeIejGeIejHeIejAeBejCeDejJeKejLeMiZ dGdHZNgdIZOdJdKZPdLdMZQdNdOZRdPdQZSGdRd d ZTGdSdTdTeZUdS)XN)tree_map tree_flattentree_unflatten) ModuleTracker)AnyOptionalUnionTypeVarCallable)Iterator) ParamSpec) defaultdict)TorchDispatchModeprodwrapsFlopCounterModeregister_flop_formula_T_PcCst|tjr |jS|SN) isinstancetorchZTensorshape)irG/var/www/auris/lib/python3.10/site-packages/torch/utils/flop_counter.py get_shapes r flop_registrycstddfdd }|S)N)out_valcs(tt|||f\}}}|d|i|S)N out_shape)rr)r!argskwargsr"frrnfszshape_wrapper..nfrr&r'rr%r shape_wrappersr)Freturncs,dtttfdtttfffdd }|S)N flop_formular*cs,stfdd}tjj|S)NcsHt|tjjstd|dt||tvrtd|t|<dS)Nzlregister_flop_formula(targets): expected each target to be OpOverloadPacket (i.e. torch.ops.mylib.foo), got z which is of type zduplicate registrations for )rrZ_opsZOpOverloadPacket ValueErrortyper RuntimeError)targetr+rrregister(s z=register_flop_formula..register_fun..register)r)rutilsZ_pytreeZ tree_map_)r+r1get_rawtargetsr0r register_fun$s   z+register_flop_formula..register_fun)r rr)r5r4r6rr3rr#s()r"c Os,|\}}|\}}||ksJ||d|S)zCount flops for matmul.r) a_shapeb_shaper"r#r$mkk2nrrrmm_flop9s r>cK t||S)zCount flops for addmm.r>Z self_shaper8r9r"r$rrr addmm_flopDs rBc KsD|\}}}|\}}} ||ksJ||ksJ||| d|} | S)z"Count flops for the bmm operation.r7r) r8r9r"r$br:r;b2r<r=floprrrbmm_flopIs    rFcKr?)z&Count flops for the baddbmm operation.rFrArrr baddbmm_flopVs rHc Ks t||S)zCount flops for _scaled_mm.r@) r8r9Z scale_a_shapeZ scale_b_shapeZ bias_shapeZscale_result_shapeZ out_dtypeZuse_fast_accumr"r$rrr_scaled_mm_flop]s rIx_shapew_shaper" transposedc CsL|d}|r|n|dd}|^}}} t|t||||d} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr7Nr) rJrKr"rLZ batch_sizeZ conv_shapeZc_outZc_inZ filter_sizerErrrconv_flop_countns   rMc Ost||||dS)zCount flops for convolution.rL)rM) rJrK_bias_stride_padding _dilationrLr"r#r$rrr conv_flopsrSc Csdd} d} | drt| d}| t|||| 7} | drIt| d}|r9| t| || || |dd7} | S| t| || || |dd7} | S)NcSs |d|dgt|ddS)Nrrr7)list)rrrrts zconv_backward_flop..trrFrN)rrM)grad_out_shaperJrKrOrPrQrRrLZ_output_paddingZ_groupsZ output_maskr"rU flop_countZgrad_input_shapeZgrad_weight_shaperrrconv_backward_flopsF    rXcCs|\}}}}|\}}} } |\} } } }||kr| kr8nJ||kr)| kr8nJ|| kr8| | kr8|| ks:Jd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rrG) query_shape key_shape value_shaperChs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   P""rhcOs t|||S)Count flops for self-attention.rh)rYrZr[r"r#r$rrr sdpa_flops rkcCsRddlm}ddlm}t|||fs|jjdkr|S|g| ddS)z If the offsets tensor is fake, then we don't know the actual lengths. In that case, we can just assume the worst case; each batch has max length. r) FakeTensor)FunctionalTensormetar) Ztorch._subclasses.fake_tensorrlZ#torch._subclasses.functional_tensorrmrdevicer-difftolistsize)offsetsmax_lenrlrmrrr_offsets_to_lengthss   ru)grad_out.ccs&|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}} } |j\}} } |j\}} }|dus;J|dusAJ|j|jksIJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}|durt|nd}||||fVqXdS|j|j|j|dur|jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrruzip)querykeyvaluerv cum_seq_q cum_seq_kmax_qmax_k_h_qr^h_kd_kh_vrfZ seq_q_lengthsZ seq_k_lengthsZ seq_q_lenZ seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shaperrr%_unpack_flash_attention_nested_shapes*s*          &rccs,|durt|jdksJt|jdksJ|dus#|j|jks#J|j\}}} } |j\}}} } |j\}}} }|dus>J|dusDJ|j|jksLJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}|durw|nd}||||fVq[dS|j|j|j|dur|jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrrx)r{r|r}rv cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krrr^rrrrfZ seqlens_qZ seqlens_klen_qZlen_krrrrrrr)_unpack_efficient_attention_nested_shapesXs*       &rT)r4c Os(t|||||||d} tdd| DS)ri)r{r|r}r~rrrcs$|] \}}}}t|||VqdSrrj.0rYrZr[rrrr    z0_flash_attention_forward_flop..rsum) r{r|r}r~rrrr"r#r$sizesrrr_flash_attention_forward_flop rc Os(t|||||||d} tdd| DS)ri)r{r|r}rrrrcsrrrjrrrrrrz4_efficient_attention_forward_flop..rr) r{r|r}biasrrrrr#r$rrrr!_efficient_attention_forward_floprrcCsVd}|\}}}}|\} } } } |\} }}}|\}}}}|| kr)| kr)|krBnJ|| kr;|kr;|krBnJ|| ksDJ||krP| |krP||ksRJd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)NrrG)rVrYrZr[rgrCr\r]r^r_r`rarbrcrdrerfZ_b4Z_h4Z_s4Z_d4rrrsdpa_backward_flop_counts    T"""""rcOst||||S)z(Count flops for self-attention backward.r)rVrYrZr[r"r#r$rrrsdpa_backward_flopsrc O*t|||||||| d} tdd| DS)N)r{r|r}rvr~rrrcs&|]\}}}}t||||VqdSrrrrYrZr[rVrrrr   z1_flash_attention_backward_flop..r) rvr{r|r}outZ logsumexpr~rrrr#r$shapesrrr_flash_attention_backward_flop rc Or)N)r{r|r}rvrrrrcsrrrrrrrr&rz5_efficient_attention_backward_flop..r) rvr{r|r}rrrrrrr#r$rrrr"_efficient_attention_backward_flop rrcCst|ts|fS|Sr)rtuple)xrrrnormalize_tupleAs r)KMBTcCs0tdtttdtt|dd}t|S)Nrrr7rw)maxminrysuffixesstr)numberindexrrrget_suffix_strJs(rcCs&t|}|d|d}|t|S)Niz.3f)rr)rsuffixrr}rrrconvert_num_with_suffixQs  rcCs|dkrdS||dS)Nr0%z.2%r)numdenomrrrconvert_to_percent_strXs rcstfdd}|S)Ncst|\}}|}t||Sr)rr)r#Z flat_argsspecrr%rrr'^s  z)_pytreeify_preserve_structure..nfrr(rr%r_pytreeify_preserve_structure]src seZdZdZ    ddeeejje ejjfde de dee e e fffd d Zd e fd d Zd e ee e e fffddZdddZddZddZddZZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() Nr7Tmodsdepthdisplaycustom_mappingcstttdd|_||_||_d|_|duri}|dur&tjdddit dd| D|_ t |_ dS)NcSsttSr)rintrrrrsz*FlopCounterMode.__init__..zs*z,FlopCounterMode.__init__..) super__init__r flop_countsrrmodewarningswarnr itemsr mod_tracker)selfrrrr __class__rrr{s  zFlopCounterMode.__init__r*cCst|jdS)NGlobal)rrvaluesrrrrget_total_flopsszFlopCounterMode.get_total_flopscCsdd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. cSsi|] \}}|t|qSr)dictrrrrrsz3FlopCounterMode.get_flop_counts..)rrrrrrget_flop_countss zFlopCounterMode.get_flop_countsc s|durj}|dur d}ddl}d|_gd}g}tdfdd}tjD]}|dkr;q4|d d }||krGq4|||d }| |q4djvroso|D] } d | d| d<q]|dd|}t |dkrzgd g}|j||d dS)Ni?BrT)ModuleZFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]\}}||dt|t|t|gq,|S)N z - )rrrappendrrrr)mod_namerrgpaddingrr;rZ global_flopsZ global_suffixZis_global_subsumedrrr process_mods  z.FlopCounterMode.get_table..process_modr.rr)r0r)leftrightr)headersZcolalign) rtabulateZPRESERVE_WHITESPACErrsortedrkeyscountextendry) rrrheaderrrmodZ mod_depthZ cur_valuesr}rrr get_tables6   zFlopCounterMode.get_tablecCs,|j|jt||_|j|Sr)rclearr __enter___FlopCounterModerrrrrrs    zFlopCounterMode.__enter__cGsD|jdusJ|jj|}d|_|j|jr t||j|Sr)r__exit__rrprintrr)rr#rCrrrrs  zFlopCounterMode.__exit__cCsV||jvr)|j|}||i|d|i}t|jjD] }|j|||7<q|S)Nr!)r setrparentsr)rZ func_packetrr#r$Zflop_count_funcrWparrrr _count_flopss  zFlopCounterMode._count_flops)Nr7TNr)__name__ __module__ __qualname____doc__rr rnnrrTrboolrrrrrrrrrr __classcell__rrrrrgs* = c@s$eZdZdefddZdddZdS) rcountercCs ||_dSr)r)rrrrrrs z_FlopCounterMode.__init__rNcCs0|r|ni}|tjjjjtjjjjtjjjjtjjjjtjjjjtjjj jtjjj jtjjj jtjjj jtjjj jtjjjjtjjjjtjjjjtjjjjhvrRtS||jjvr|tjjjjur||j|i|}|turx|WdSWdn1swY||i|}|j|j|||Sr)ropsatenZ is_contiguousdefaultZ memory_formatZis_strides_like_formatZis_non_overlapping_and_denserrZsym_sizeZstrideZ sym_strideZstorage_offsetZsym_storage_offsetZnumelZ sym_numeldimZprimZlayoutNotImplementedrr roZ decomposerZ_overloadpacket)rfunctypesr#r$rrrrr__torch_dispatch__s6               z#_FlopCounterMode.__torch_dispatch__)rN)rrrrrr rrrrrsr)Fr)NNNFN)VrZtorch.utils._pytreerrrZmodule_trackerrtypingrrr r r collections.abcr Ztyping_extensionsr collectionsrZtorch.utils._python_dispatchrmathr functoolsrr__all__rrrrrr r__annotations__r)rmmrr>ZaddmmrBZbmmrFZbaddbmmrHZ _scaled_mmrIrTrrMZ convolutionZ _convolutionrSZconvolution_backwardrXrhZ'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionZ#_scaled_dot_product_cudnn_attentionrkrurrrZ_flash_attention_forwardrZ_efficient_attention_forwardrrZ0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardZ,_scaled_dot_product_cudnn_attention_backwardrZ_flash_attention_backwardrZ_efficient_attention_backwardrrrrrrrrrrrrrs        *    ' g6 36 0