a hV}@s UddlZddlZddlZddlTddlmZddlZddlmmZ ddl m Z ddl m Z ddlmZgZeeed<iZeeefed <dd d ZdddZdddZeddddZeeeeddddZddZddZddZeeeZ ee!ddd Z"d!d"Z#d#d$Z$d%d&Z%d'd(Z&e ej'j(j)j*ej'j(j+j*ej'j(j,j*ej'j(j-j*ej'j(j.j*ej'j(j/j*ej'j(j0j*gd)d*d+Z1e ej'j2j3j*d)d,d-Z4e ej'j(j5j*gd)d.d/Z6e ej'j(j7j*d)d0d1Z8e ej'j(j7j9d2e8e ej'j(j:j*d3d4d5Z;e ej'j(jj*d9d:d;Z?e ej'j(j@jAd<d=d>ZBe ej'j(jCj*d)d?d@ZDe ej'j(jEj*dAdBdCZFe ej'j(jGj*d)e$e ej'j(jHj*ej'j(jIj*ej'j(jJj*ej'j(jKj*ej'j(jLj*gd)dDdEZMe ej'j(jNj*dFeMe ej'j(jOj*dGeMe ej'j(jOjPdHeMe ej'j(jQj*d)dIdJZRe ej'j(jSj*dKdLdMZTe ej'j(jUj*dNdOdPZVe ej'j(jWj*dQdRdSZXe ej'j(jYj*dTdUdVZZe ej'j(j[j\dWdXdYZ]e ej'j(j[j*dZd[d\Z^e ej'j(j_j`d]d^d_Zae ej'j(jbj*d`dadbZce ej'j(jdj*dcdddeZde ej'j(jej*dfdgdhZfe ej'j(jgjhdidjdkZie ej'j(jjj,dldmdnZke ej'j(jlj*dodpdqZme ej'j(jnj*drdsdtZoe ej'j(jpj*dudvdwZqe ej'j(jrj*dxdydzZse ej'j(jtj*d{d|d}Zue ej'j(jvj*d~ddZwe ej'j(jxj*dddZxe ej'j(jyj*dddZye ej'j(jzj{dddZ|e ej'j(j}j*dddZ~e ej'j(jj*dddZe ej'j(jj*dddZddZe ej'j(jj*dZddZe ej'j(jjdddZe ej'j(jjhdddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*gdddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jjhdddZe ej'j(jj`dddZe ej'j(jj*de ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj,dddZe ej'j(jj*dZddZe ej'j(jjdddZe ej'j(jj,dddZe ej'j(jjdddZe ej'j(jj,dddZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gd)dd„Ze ej'j(jj,dddĄZe ej'j(jj,dddƄZe ej'j(jj*dddȄZe ej'j(jj*dddʄZe ej'j(jj*ddd̄Ze ej'j(jj*ddd΄Ze ej'j(jj*dσddфZe ej'j(jj*drddӄZe ej'j(jj*dԃddքZe ej'j(jj*d׃ddلZe ej'j(jj*ej'j(jj*gd)ddۄZe ej'j(jj*d)dd݄Ze ej'j(jj*dރddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*d)ddZe ej'j(jj*d)ddZe ej'j(jj*d)ddZe ej'j(jj*d)ddZe ej'j(jj*d)ddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jjېdddZe ej'j(jjېdddZe ej'j(jj`d)ddZe ej'j(jj*ddd Zdd lmZmZdd lmZeedej`ej`ej`e!eeeeefeeeej`ej`fd ddZeed ej`ej`ej`ej`ej`ej`ej`ee!efeeeeeefeeeej`ej`ej`eeej`dffdddZe ej'j(jj*dddZejddFZ(e(dede(dede(dedWdn1 s0YdS(!N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention NestedTensor__all__JAGGED_OPS_TABLEFcsddlm}t|ttfrHt|fdd|D}t|t|S|rV||}|dkrf|ksjJ|dkrzdS|dS)Nrcanonicalize_dimsc3s|]}t|VqdSN_outer_to_inner_dim.0dndim ragged_dimH/var/www/auris/lib/python3.9/site-packages/torch/nested/_internal/ops.py z&_outer_to_inner_dim..r)torch._prims_commonr isinstancetuplelisttypedictfromkeys)rdimr canonicalizer outputrrrrs  rTc Csxddlm}|||}||kr2|s2t|dn|dkrL|sLt|d|r\t|||n|} |rt|dk} | | fS| S)Nrr z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)rr RuntimeErrorr) rr!rop_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr wrappedretoperating_on_batchrrr_wrap_jagged_dim's    r,c sddlmt|ttfs,Jdt|fdd|D}d|v}|v}tfdd|D}ttfdd|D}||||fS) z For NestedTensor operators, wraps dimensions to non-negative values, and returns metadata related to reduction dimension(s). rr z<_wrap_jagged_dims(): cannot iterate over dimensions of type csg|]}|qSrrr)r rrr Psz%_wrap_jagged_dims..c3s|]}|dko|kVqdS)rNrr) ragged_idxrrrVrz$_wrap_jagged_dims..c3s|]}t|VqdSr rr)rr.rrrZr) rr rrrranyrr ) rdimsr%r.Z wrapped_dimsZoperate_on_batchZoperate_on_raggedZoperate_on_non_batchZouter_to_inner_dimr)r rr.r_wrap_jagged_dimsDs   r1) schema_strreturnc s|d}dd|Dd}t||}|ddkrD|dd}nNt||kr`t|t|kstd|jd|d |d t|d t|d d dddddddd}t|D]\}} | d\} } | d} | r| ddn| } | |vrtd| |t|kr8| std|jd|d| q|| | ffdd }|||sdddddd}td|jd|d| d || qdS)!N, cSsg|]}|dqS)?)endswithrxrrrr-brz check_schema..T... NestedTensor (z): expected at least z arguments and at most z arguments, but got: z argumentscSst|tjot|t Sr )rtorchTensorrr8rrrqrzcheck_schema..cSst|to|jduo|jdkSNr)rr_lengths _ragged_idxr?rrrr@rs cSs t|tSr rrr?rrrr@uscSsdS)NTrr?rrrr@xr)tjtjt_allr/z: r5zUnknown arg type: z) missing required argument: cs |r|dup|S|SdSr r)r8 is_optionalZ _check_fnrrcheck_fnszcheck_schema..check_fnZtensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz )rEzt?rFrGr/z ): expected z to be a ) splitcountlen ValueError__name__ enumerater6keysAssertionError)r2funcargskwargsZnamed_arg_typesZnum_optional_argsZmin_argsZarg_type_check_fnsiZnamed_arg_typenameZarg_typerHZnormalized_arg_typerJZ type_to_descrrIr check_schema`s\      rX)aa_namebb_namer3cCs:|j|j|j|jkr6td|jd|d|ddS)Nr;z : expected  and z' to have the same exact offsets tensor.)_sizerCr$rO)rSrYrZr[r\rrrcheck_ragged_dim_samesr_cCsL|jd}|jd|}|d|}t|t|koJtddt||DS)Nrcss"|]\}}||kp|dkVqdS)r9Nr)rnssrrrrrz%raggedness_matches..)rCr^rMallzip)ntsizeendZ nt_raggedZ size_raggedrrrraggedness_matchess   rgcCs*|dkr&|jddkr&|d}q|SNrr)r!shapesqueeze)rErrrsqueeze_leading_oness rkcs4ttsgtts gfdd}|S)Ncs2D](}fdd}D]}||||<qqS)Ncsfdd}|S)Ncs.tg|Ri|g|Ri|Sr )rX)rTrU)aten_oprSr2rrinnersz@register_func..wrapper..get_inner..innerr)rlrm)rSr2)rlr get_innersz1register_func..wrapper..get_innerr)rSrlrntableaten_opsr2tablesrSrwrappers zregister_func..wrapper)rr)rrrqr2rtrrpr register_funcs   ru)r3c Os*t|d}|dur|Stjj|jvr&ddlm}|D]"}||r8td|j d|q8t dd|D}|dkrg}|j j D]<}t |jtjr||jdqq||jd q|d d |}t||g|Ri|tt|S|d kr&td |g|Ri|tt|SdS)Nr is_nested_intr;z: invalid argument css|]}t|tjVqdSr )rr=r>r7rrrrrz lookup_jagged..rz: jt_allz: anyr:r4zlhs: any, rhs: any, ...)r getr=TagZ pointwisetags%torch.fx.experimental.symbolic_shapesrwr$rOsumZ_schema argumentsrrZ TensorTypeappendrWjoinrX functoolspartialjagged_unary_pointwisejagged_binary_pointwise) rSrTrUZ dispatch_funcrwargZnum_tensor_argsZ schema_partsZcheck_schema_strrrr lookup_jaggeds4        rcCs|||j|jd}|S)Noffsetslengths_metadata_cacherCr)rrUrrrextract_kwargss rcs>tdd|Dt|fdd|Di|fitS)Ncss|]}t|tr|VqdSr rDrrrrrrrz)jagged_unary_pointwise..c3s |]}|ur|jn|VqdSr _valuesrnjtrrrrnextrrrSrTrUrrrrs rcsZ|d|d}}t|ts*t|ts*Jd}t|trt|trt||jrt||j|jg|ddRi|fit|St||j|j|jt|t}|rt|nt|}|r||fn||f\}| krt dt |} | dkrP|rj| fn| jf\} } t|| | g|ddRi|fi|S| | kr>|j d|j dkrt||j|j |j ddl m} j} j}|jj jd}dur||fddt|j D}| |jj|| |d }|r|fn|f\} } || | g|ddRi|St||j|j |j dS) NrrzHcannot call binary pointwise function {} with inputs of shapes {} and {}rxz+NYI: broadcasting NT with T with larger dimnested_from_paddedcs"g|]\}}|jkr|nqSrrC)rrVr8rd padded_max_Srrr-^rz+jagged_binary_pointwise..rr.sum_S min_seqlen max_seqlen)rrrgr^rrr$formatrOr!NotImplementedErrorrkri nested_tensorr_maybe_min_seqlen_maybe_max_seqlenrCexpandrP_offsets)rSrTrUrYr[Zmismatch_error_msgZa_is_ntZextracted_kwargsrEZ t_squeezedlhsrhsrrrtotal_LZt_as_ntrrrr$s^ " ,    rcOs|tjjjurt|i|S|jdkrR||djg|ddRi||dS|jdkrddd}t|||dd \}}|d }t | |d |j dd d }t | |d|j dd d }||kr|St tj|j||d} g|jd|| |j|ddR} |j| S|jdkrddd} t| ||dd \}}|d }|d} | |j d} t| | krtdtj||i|WdS1s0Yt|dS)NZapply_rrflattenr9cSsdSr r)input start_dimend_dimrrr _flatten_sigsz+jagged_torch_function.._flatten_sigTrTrUZnormalize_to_only_use_kwargsrrF)r&rZrms_normcSsdSr r)rnormalized_shapeweightepsrrr _rms_norm_sigsz,jagged_torch_function.._rms_norm_sigrzNrms_norm(): Normalization over the ragged dim not supported for nested tensors)rr9)NN)r=Z_CZ_nnZscaled_dot_product_attentionrrOrrpopr,r!rCrreduceoperatormulrireshaperMrNZDisableTorchFunctionSubclassr)rSrTrUr_ new_kwargsinprrproductZ new_shaperrZmax_normalizablerrrjagged_torch_functionrsZ $    *       .rz self: jt_allcOs|tjjjjkrdS|tjjjjkr.|djS|tjjjjkrLt|djS|tjjj jtjjj jfvr|dj durt t |dj t|djddS|dj S|tjjjjkr|djS|tjjjjkr|djSdS)NFrrx)r=opsatenis_non_overlapping_and_densedefaultsym_sizer^r!rM sym_numelnumelrBintr}mathprodr sym_strideZ_stridessym_storage_offsetZstorage_offsetrrrrtensor_attr_supported_getters  * rcOstjSr )r=Zjaggedrrrrprim_layout_defaultsrcOs|tjjjjkrtddS)NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)r=rrrerr$rrrrtensor_attr_unsupported_gettersrcOspddlm}t|||dd\}}|d}|dur:dS|dtj|d<|dtjkr^dS||j fi|S)Nr)is_contiguous_for_memory_formatTrrF memory_format) rrrrrryr=contiguous_formatZpreserve_formatr)rSrTrUrrrrrrris_contiguous_generals    rz!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c Ost|||dd\}}|d}t|}|jdurt|dtjkrtddlm}|jdks\Jd|| dd\}}|St ||j fi|fi|S) NTrrrr)jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r) rrrrBr=rrrrCunbindrr) rSrTrUrrrZnew_metarZcontigrrr clone_defaults     rzinput: jt, weight: t, bias: t?cOs@t|||dd\}}|d}t||jfi|fit|SNTrrrrrrrrSrTrUrrrrrrlinear_default#s   rz6self: jt, grad_output: jt, weight: t, output_mask: anycOst|||dd\}}|d}|d}|d}|d}d\} } } t||d|d|d r|tt|j|fit|} |d r|jd | d } |jd | d } t| | } |d rt t |jj d }|d kr|j} ntj|j|dd} | | | fS)NTrr grad_outputr output_mask)NNNselfrrr9rxrF)keepdim)rrr_rr=matmulrrrrerErrangercloner})rSrTrUrrrrrrZdsZdwdbZgrad_2dZinput_2dZ reduce_dimsrrrlinear_backward_default.s2        rzinput: jt_all, dtype: anycOs@t|||dd\}}|d}t||jfi|fit|Srrrrrrto_dtypeWs   rcOsddlm}t|||dd\}}|d}|d||jfi|}|jj|jd}d} |jdurt|jj|jd} dd l m } dd l m } m } |jdur|jn|j} | dur|n| }t|| | fr| |}| | }|j|_n || ||<t|}||d <| |d <t|fi|}|S) Nr_tensor_symint_registryTrrlayoutdevicer FakeTensorFunctionalTensormb_unwrap_functional_tensorrr)rrrrrrtorrBtorch._subclasses.fake_tensorr#torch._subclasses.functional_tensorrrrnested_int_memorr)rSrTrUrrrr new_values new_offsets new_lengthsrrr ragged_source new_thingtgtsrc inp_kwargsr#rrrto_copy_defaultbs2        rz-self: jt_all, src: jt_all, non_blocking: any?c Ost|||dd\}}|d}|d}|j|jkr|}dd|D}|} dd| D} || krtd|jd |jt|| D]\} } | | q|j|j|S) NTrrrcSsg|] }|jqSrrircrrrr-rz copy_default..cSsg|] }|jqSrrrrrrr-rz 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorrr9z-inf) max_lengthsZ padding_valuer!r)rrrr$rr1r!rCrBr=nn functionalrrr_jagged_to_padded_dense_forwardrrrir _max_seqlenfloat_padded_dense_to_jagged_forwardrr) rSrTrUrrrreduce_on_batchreduce_on_raggedZ_reduce_on_non_batchZpadded_softmax_valuesZsoftmax_valuesrrr_softmax_defaultsv     r z7grad_output: jt, output: jt, dim: any, input_dtype: anycOsNt|||dd\}}|d}|d}t||j|jfi|fit|S)NTrrr#r)rSrTrUrrgrad_outr#rrr_softmax_backwardLs   r z!self: jt, float: any, train: any?cOs\t|||dd\}}|d}||jfi|\}}t|fit|t|fit|fSrrrrrr)rSrTrUrrrZout1Zout2rrrnative_dropout_default[s  rz%grad_output: jt, mask: jt, scale: anycOsNt|||dd\}}|d}|d}t||j|jfi|fit|S)NTrrmaskr)rSrTrUrrrrrrrnative_dropout_backward_defaultks   rz2self: jt_all, dim: any, keepdim: any?, dtype: any?cOst|ddg|Ri|S)Nrr_apply_reductionrrrr prod_dim_int{srzself: jt_all, dtype: any?cOs0t|||dd\}}|d}||jfi|Srrrrrr prod_defaults   rz$self: jt, split_size: any, dim: any?cs^t|||dd\}}|dt|djd|d<tfdd|jfi|DS)NTrrr!rKc3s$|]}tfd|itVqdS)valuesNrrr7rrrrszsplit_tensor..)rrr,r!rCrrrSrTrUrrrrr split_tensors   rz%self: jt, split_sizes: any, dim: any?csZt|||dd\}}|dt|djd|d<fdd|jfi|DS)NTrrr!split_with_sizescs"g|]}tfd|itqSrrr7rrrr-sz,split_with_sizes_default..)rrr,r!rCrrrrrsplit_with_sizes_defaults   rz+self: jt, dim: any, start: any, length: anycOsdt|||dd\}}|d}t||d|jd}||j||d|dd}t|fit|S) NTrrr!narrowstartlengthr!rr )rrr,r!rCrrr)rSrTrUrrrr!rrrrrs  rz self: jt, chunks: any, dim: any?c st|||dd\}}|dt|djddd\|d<}|r|d}j}||}dd |D} d d | D} fd d | Dd d |D} j | fd d t dt DSfdd |jfi|DSdS)NTrrr!chunkr(chunkscSsg|]}tj|ddqS)rr)r=cumsumr7rrrr-rz!chunk_default..cSsg|]}tj|dddqS)rrr)value)Fpadr7rrrr-rcsg|]}|jdqS))rrCr)rZ per_offsetsrrrr-scSsg|]}|qSr)r}itemr7rrrr-rcs&g|]}tfd|i|qSrrrrV) chunk_values nested_kwargsrrr-srcs"g|]}tfd|itqSrrr7rrrr-s) rrr,r!rCrdiffr"rrKrrM) rSrTrUrrr+r$rZchunked_lengthsZchunked_offsetsZ split_sizesr)r,rr-r chunk_defaults0         r/zself: jt_all, dim: any?c st|||dd\}}|d}|dkr,td|d}||}|}|jdttt ttdfdd } |dur| | t j d d Sdkrtd | | | fd dt|jdDS)NTrr!rz2unbind(): only supported for NestedTensor on dim=0r)rBrcsd}jd}tt|D]X}t||t|||k|||7}|durt|||||kddqt||k|durtt|D]$}t||t|||kqdS)NrrcSsdS)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionrrrrrr@rz2unbind_int.._torch_check..)rirrMr=Z_check_is_size_check)rBrZ lengths_sumZragged_dim_sizerV)r.rrr _torch_checks  z unbind_int.._torch_checkrrzAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)cs*g|]"}tjd||dqS)rr!)r=rr+Zlengths_scalarsZoffsets_scalarsr.rrrr-1szunbind_int..)N)rr$rrrrrCrrrr.tolistr=rKrri) rSrTrUrrr!rrrr1rr2r unbind_ints4  $   r4zself: jt, dim: anycOsbt|||dd\}}|d}|j}tt|j|d|jd|d<t||fi|fit|S)NTrrr!rj) rrrr,rMr^rCrr)rSrTrUrrrrrrr squeeze_dim<s  r5zself: jt_all, dim: anyc Ost|||dd\}}|d}|j}|d}tt|jd||jddd|d<t|}|d|jdkrx|dd7<t||fi|fi|S) NTrrr!r unsqueeze)r'rC) rrrr,rMr^rCrr) rSrTrUrrrrr!rrrrunsqueeze_defaultKs   r7ztensors: any, dim: anycst|||dd\}}|d}dd|D}t|dks.rcs g|]}|jr|n|qSr)r9 expand_asr:firstrrr-nrr!catcSsg|] }|jqSrrr:rrrr-wr)rrrMr,rirCrr)rSrTrUrrr8nestedr!rr<r cat_defaultbs"   r@zself: any, other: anyc sDt||dd\}}|d}|d}fdd}fdd}|jr|js|d kr||kr|j|d kr|||S|d kr||kr|j|d krt|j|fi|fit|SnN|jsr|jrr|d kr ||kr |jd kr |||S|d kr(||kr(|jd kr(t||jfi|fit|Sn|jr(|jr(|d kr|d krt||j rt|j|jfit|S|d kr(|d kr(|jd kr(|jd kr(| |j| |jkr(t |||St d |j d |jdS)NTrrothercs fddt||DS)Ncsg|]\}}||qSrr)rZa_compZb_comprsrrr-sz8matmul_default.._unbind_impl..)rcr)rYr[rsrr _unbind_impls z$matmul_default.._unbind_implc s|jr |}n|}ddlm}|j}|j}|}|jj|jd}|durJ|}g|jd|j||j|jddR}|jd|d} |jr| |} n || } || |j |j|||dS)Nrr output_sizer) r9rrrrrrirCto_padded_tensorr) rYr[rdrrrrr padded_shapeZ padded_ntZpadded_trsrr _padded_impls:   z$matmul_default.._padded_implrrxz1matmul(): not supported between inputs of shapes r])rrr9r!rCrrrrgr^rer=stackr$ri) rSrTrUrrrrArBrHrrsrmatmul_default{sh     %     , ,*  rKzself: jt_all, mat2: anycOsdt|||dd\}}|d}|d}|dkr cs"g|]}|jkrdn|qS)r9rrrrerrr- rz"expand_default..r) rrrgr$r^rr!rrr)rSrTrUrrZ expand_argrrNrexpand_defaults   rOzself: t, other: jtcOsDt|||dd\}}|d}|d}t|||jfit|S)NTrrrArrLrrrexpand_as_default s    rPzself: jt_all, size: anycOspt|||dd\}}|d}|d}t||krd|gddt|t|D|StddS)NTrrrecss|] }dVqdS)rNr)rrrrrr#rzbroadcast_to..zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)rrrMr!rrrN)rSrTrUrrrrerrr broadcast_tos   ,rQz tensors: anyc Ost|||dd\}}|d}t|dkr2tdt|dkrF|dSg}tjdd|D}td d|D}|D]Z} | jr|| |qt| t|kr|t | |j j fit|qttd qtt|S) NTrr8rz7broadcast_tensors(): expected at least one tensor inputrcss|] }|jVqdSr rr:rrrr8rz$broadcast_tensors..css|]}t|tr|VqdSr rDr:rrrr:rzubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)rrrMrNr=Zbroadcast_shapesrr9rrQr!rrrirr) rSrTrUrrr8ZoutsZbroadcast_shaperrErrrbroadcast_tensors+s,    rRz(condition: jt_all, self: any, other: anycOspt|||dd\}}|d}|d}|d}t|||\}}}t||j|j|jfi|fit|S)NTr conditionrrA)rrr=rRrrr)rSrTrUrrrSrrArrr where_selfKs    rTzself: jt, device: any?cOs@t|||dd\}}|d}t||jfi|fit|Srrrrrr_pin_memory_default`s   rUcOs0t|||dd\}}|d}||jfi|Srrrrrris_pinned_defaultks   rVzself: jt_all, other: jt_allcOs|dj|djkSrh)r^rrrris_same_size_defaultvsrWcsZt|||dd\}}|d|ddupJt|dttfoJt|ddk}|r|jfi|}|ddrt|ttfrt|fdd |D}n | j }|S|d} t|dttf} | s| g} t | |j \} } } }| s| d} | |d<| rj durt|d dd lm}| r| rh|jfi|}|ddrd|d d |}|S|r|t|d|d| rj gnj }||fd|i|Sn| rt|d|jfi|}t|ddsDt|dttfr|dn|dg}|D]&}|j dkrdd8<q|fdd |SdS)NTrrr!rrFc3s|]}|jVqdSr )r6rC)rorrrrrz#_apply_reduction..zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapcSs |dS)Nr)r6rXrrrr@rz"_apply_reduction..zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsrrCcst|fiSr rrZ) out_kwargsrrr@r)rrrrrrMrryrr6rCr1r!rBr$Ztorch.utils._pytreerYrFr)rS func_nameZidentity_elementrTrUrrZfull_reductionoutZdim_to_convertZ is_dimlistZ converted_dimr r reduce_on_non_batchrYZ dim_to_passZdimlistrr)rr[rr}s         rcOs0t|||dd\}}|d}||jfi|Srrrrrr sum_defaults   r_z3self: jt_all, dim: any?, keepdim: any?, dtype: any?cOst|ddg|Ri|S)Nr}rrrrrrsum_dim_IntListsr`z"self: jt_all, dim0: any, dim1: anyc Os.t|||dd\}}ddlm}|d}|||d|df\}}||jks\||jkr|dksl|dkrttd||jkr|} n|} t|} | | d <t| t t |j ||jt t |j ||jfi| St||d|jd |d<t||d|jd |d<t||jfi|fit|S) NTrrr rdim0dim1z?Transpose is not supported on the batch dimension for jagged NTrC transpose)rrr rr!rCrNrrrrcrrMr^r,r) rSrTrUrrr rrarbZto_dimrrrr transpose_ints>    rdzself: jt_all, dims: anyc st|||dd\}}|d|d}t}tjt|krftdddt|dd d lm}||}t|tt|krtd j durtd |d d krtd | j |d<fdd|ddD} | |d<t |j fi|fi|S)NTrrr0z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().rr z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrCcsg|]}t|jqSr)rrC)rr!rZinp_dimrrr-Hsz#permute_default..r)rrrrMr^rNrr setrBindexrCrr) rSrTrUrrr0rr Zcanonicalized_dimsZ inner_dimsrrfrpermute_default's>            ricst|||dd\}}|d|djdkr\tjtkr\tdjddtd ksrtstd jd fd d fddttdD}t  ,t |j |fitWdS1s0YdS)NTrrrerzVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (rerIzview(): cannot view shape z as cs*|jdkrj|S|dSdSrA)rCrre)Z inner_idxrNrrget_inner_sizets z$view_default..get_inner_sizecsg|] }|qSrrr+)rjrrr-{rz view_default..)rrrCrr^r$rMrgrr=Zinference_modeZ is_inferencerrr)rSrTrUrrZ inner_sizer)rjrrer view_defaultPs&   rkzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anycOst|||dd\}}|d}|dkr2td|d}|j|j}|t|}|dkrftd||vr|jdurtd ||vrtj j j |j j |jd |jg|jgd } tj j j tj|j jdd f|j|jd |jg|jgd | j} |jd d | jd} tj| ddd| } | | | } tjt| ddd| }t||d}| |}tj j j||jg|j jddd|j|jd d}t|fit|| |fS||j fi|\}} }t|fit|| |fS)NTrrrxzOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsrrzVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr)rr)rdtype)rrx)r!rrrr9)rrr!r$rirCrMrBr=rrrrrrrZonesrrmrr.r6r}Zsquaresqrtr unflattenrr)rSrTrUrrrrZ ragged_sizeZnum_dims_not_normalizedZ padded_inputZ padded_maskZragged_lengthsmeanZpadded_normalizedZvarianceZstdZpadded_layer_normZjagged_layer_norm_valuesr#rrrnative_layer_norm_defaults        rqzpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc Ospt|||dd\}}|d}|d}||j|jfi|\}}} |durVd|| fSt|fit||| fS)NTrr rr) rSrTrUrrr rZd_inputZd_gammaZd_betarrr"native_layer_norm_backward_defaults    rrz"self: jt_all, dim: any, index: anycOst|||dd\}}|d}t||d|jddd\|d<}|rV||dS|jdurhtdt|}|d|jd kr|d d 8<t ||j fi|fi|S) NTrrr!selectr#rhzSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesrrC) rrr,r!rCrrBrNrrr)rSrTrUrrrr+r[rrr select_ints"   rtz7self: jt, dim: any?, start: any?, end: any?, step: any?cOs\t|||dd\}}|d}t||d|jd|d<t||jfi|fit|S)NTrrr!slice)rrr,r!rCrrrrrrr slice_tensors  rvz8input: jt_all, indices: any, values: t, accumulate: any?cOst|||dd\}}|d}|d}t||ks.sizesr!"value_selecting_reduction_backwardrrC) r|rwrrrrrPr,rMrerCrr) rSrTrUrrrrwr.rrrrvr*value_selecting_reduction_backward_default=s,         rcOst|||dd\}}|d}|D]J}t|ts8td||dkrTtdt||djs"tdq"t|dd|d |dj d |d <t|d d |Dfi|fit |dS) NTrr8z+stack(): expected all nested tensors inputsrz9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurerr!rJcSsg|] }|jqSrrr:rrrr-|rz!stack_default..) rrrrr$r!rgrir,rCr)rSrTrUrrr8rErrr stack_defaultas.    rzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?cOsLt|||dd\}}|d}|d}t|||jfi|fit|S)NTrrwrr)rSrTrUrrrwrrrrembedding_defaults   rzYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anycOs>t|||dd\}}|d}|d}||j|jfi|S)NTrrwrr)rSrTrUrrrwrrrr embedding_dense_backward_defaults    rcOs(t|||dd\}}|d}|jSr)rrrdetachrrrrvalues_defaults   rcOs(t|||dd\}}|d}||jSrrrrrr all_defaults   rz-self: jt_all, padding: any, output_size: any?c Osvt|||dd\}}|d}|jdur0td|d}|durL||j}n|jdur\|jn |jd}| }|jdkr| |jdd}|j } | dkr|j dd }n| dkr|d }|jtju} | r|jr|tj}tjj||jg|g|d } | r| jr| tj} t| dkr@| d | dd} nt| dkrX| d } |jdkrr| |jd} | S) NTrrz?to_padded_tensor(): not supported for nested tensors with holesrErrrxrlr9padding)rrrBr$rCZ_max_seqlen_tensorrrrerrcrir!rr6rmr=boolis_cudarhalfrrrrrMrorj) rSrTrUrrrrEZ max_seq_lenrZ values_shapeis_boolZ padded_outrrrto_padded_tensor_defaultsR               rzcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?cOsht|||dd\}}|d|d}}|dd}|dkrF||d}|j}|dkrf|jdd }n|dkr||d }|jtj u} | r|j r| tj }tj j||g|d } | r| j r| tj } t|dkr| d |dd} nt|dkr| d } |dkr | |dd } |d } |d} i} | durF| | d <| durX| | d<t| ||| dS)NTrpaddedrr.rrIrxrlr9rrrr)rCr)rryrcrir!rr6rmr=rrrrrrrrMrorjr)rSrTrUrrrrr.Zpadded_ragged_dim1_shaperrrrmetadata_cacherrr"_nested_from_padded_tensor_defaultsN                 rzcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c Os~t|||dd\}}|d|d|d}}}|d}|d} |d} i} | dur\| | d<| durl| | d<t||||| d S) NTrrrrr.rr)rrCr)rr) rSrTrUrrrrrr.rrrrrr _nested_view_from_jagged_default6 s,  rcOs$t|||dd\}}|d}|jSr)rrrrrrr_nested_get_offsetsV s   rcOs$t|||dd\}}|d}|jSr)rrrBrrrr_nested_get_lengths` s   rcOs$t|||dd\}}|d}|jSr)rrrCrrrr_nested_get_ragged_idxj s   rcOs,t|||dd\}}|d}|jddS)NTrrrrrrryrrrr_nested_get_min_seqlent s   rcOs,t|||dd\}}|d}|jddS)NTrrrrrrrr_nested_get_max_seqlen~ s   rzself: jt, mask: anyc Ost|||dd\}}|d}|d}|jdkr||j|dSrh)rrrB)rSrTrUrstreamrrrrecord_stream_default s    rzRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?cOsLt|||dd\}}|d}t|ddkr@||jfi|StddS)NTrrrerz2new_empty() not supported for NJT with shape != ())rrrMrr$rrrrnew_empty_default s   rzself: jt_all, ...cOs:tdd|D}t|dd|Di|fit|S)Ncss|]}t|tr|VqdSr rDrrrrr rz&activation_backward..css"|]}t|tr|jn|VqdSr )rrrrrrrr rr)rSrTrUrrrractivation_backward s  rzself: jt_all, value: anycOs@t|||dd\}}|d}t||jfi|fit|Srrrrrr fill_Scalar s   rcOs4t|||dd\}}|d}||jfi||Srrrrrr fill__Scalar s   rc OsTt|||dd\}}|d}t|}||j\}}t|fi|t|fi|fSr)rrrrr) rSrTrUrrrrZmantissaexponentrrr frexp_Tensor s  rz+grad: any, self: any, other: any, mask: anyc Ost|||dd\}}|d}|d}|d}|d}|durHdSd} |drht||d d } d} |d rt|d d |} | | fS) NTrrrrAr)NNrr9r)rrr=rrc) rSrTrUrrrrrAZgrad_input_maskZ grad_selfZ grad_otherrrrmatmul_backward_default s      r)flex_attentionflex_attention_backward) GraphModuler) querykeyr' score_mod block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersr3c Cs|dkr$|dkr$|dks(Jtdd||DrFtdt|d|d|d||||||d } tjj| d dd d|j |j |j |jd  dd} tjj| d dd d|j |j |j |jd  dd} | | fS) Ncss |]}t|tjo|jVqdSr )rr=r>r9)rbufrrrrC szflex_njt..zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r)rrrrrrrrxrr)r!r/r$flex_attention_hoprr6r=r?nested_tensor_from_jaggedrcrjrrBrr) rrr'rrrrrrr#Z output_njtZ logsumexp_njtrrrflex_njt4 sJ (    r.)rrr'r] logsumexpr grad_logsumexpfw_graph joint_graphrrrrrr3cCst|d|d|d|d|d|d|d||| | | | | d}|\}}}}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}||||fS)Nr) r]rr rrrrrrrrrrxr) flex_attention_backward_hoprr6r=r?rrcrjrrBrr)rrr'r]rr rrrrrrrrr#Z dense_q_gradZ dense_k_gradZ dense_v_gradZscore_mod_other_buffer_gradsZ njt_q_gradZ njt_k_gradZ njt_v_gradrrrflex_njt_backwardm sZ        rz self: anycOsddlm}|S)Nr)_nt_view_dummy)Z$torch.nested._internal.nested_tensorr)rSrTrUrrrr_nested_get_jagged_dummy s rrZIMPLZCPUZCUDAZMeta)F)TFF)r)rr)rr)rrrtypingrr=Ztorch.nn.functionalrrr(Ztorch.fx.operator_schemasrZtorch.nested._internal.sdparrrr rAny__annotations__r Dictrr,r1strrXr_rgrkrurZregister_jagged_funcCallablerrrrrrrrrrr!rrrrrZprimrrrerrxrrrrZlinearrZlinear_backwardrrrmrZ_to_copyrrrrZ empty_likeZ ones_likerZ rand_likeZ randn_likerZ full_likeZ randint_likeZ low_dtypeZzero_rZ_softmaxr Z_softmax_backward_datar Znative_dropoutrZnative_dropout_backwardrrZdim_intrrrKr>rrrrr"r/rrr4rjr5r6r7r>r@rrKZbmmrMrrOr;rPrQrRwhererrTZ _pin_memoryrU is_pinnedrVZ is_same_sizerWrr}r_r{r`rcrdZpermuteriviewZ _unsafe_viewrkZnative_layer_normrqZnative_layer_norm_backwardrrrsrtrurvZ index_putryZ convolutionrzrpr|r}r/r0r~rrbrrrrrrrrrrrrrrrrrrJrZ embeddingrZembedding_dense_backwardrrZ_nested_get_valuesrrrFrZ_nested_from_padded_tensorrZ_nested_view_from_jaggedrrrrrrrrZ_nested_select_backwardrZ record_streamrZ new_emptyZ new_zerosZnew_onesrZ elu_backwardZhardshrink_backwardZhardsigmoid_backwardZhardtanh_backwardZsoftplus_backwardZsoftshrink_backwardrfillZScalarrZfill_rfrexprZmatmul_backwardrZ&torch._higher_order_ops.flex_attentionrrrrZtorch.fx.graph_modulerZpy_implTuplerrUnionrrlibraryZ_scoped_libraryimplrrrrs$        A   )  NH                   %  (        /    H                ) H    m            j     * ( /  a        T    &                                 ;  5                           8   ?