o wZhR}&@s" UddlZddlZddlZddlTddlmZddlZddlmmZ ddl m Z ddl m Z ddlmZgZeeed<iZeeefed <d,d d Z d-ddZd.ddZdeddfddZdededededdf ddZddZddZd d!ZeeeZ dee!fd"d#Z"d$d%Z#d&d'Z$d(d)Z%d*d+Z&e ej'j(j)j*ej'j(j+j*ej'j(j,j*ej'j(j-j*ej'j(j.j*ej'j(j/j*ej'j(j0j*gd,d-d.Z1e ej'j2j3j*d,d/d0Z4e ej'j(j5j*gd,d1d2Z6e ej'j(j7j*d,d3d4Z8e ej'j(j7j9d5e8e ej'j(j:j*d6d7d8Z;e ej'j(jj*d<d=d>Z?e ej'j(j@jAd?d@dAZBe ej'j(jCj*d,dBdCZDe ej'j(jEj*dDdEdFZFe ej'j(jGj*d,e$e ej'j(jHj*ej'j(jIj*ej'j(jJj*ej'j(jKj*ej'j(jLj*gd,dGdHZMe ej'j(jNj*dIeMe ej'j(jOj*dJeMe ej'j(jOjPdKeMe ej'j(jQj*d,dLdMZRe ej'j(jSj*dNdOdPZTe ej'j(jUj*dQdRdSZVe ej'j(jWj*dTdUdVZXe ej'j(jYj*dWdXdYZZe ej'j(j[j\dZd[d\Z]e ej'j(j[j*d]d^d_Z^e ej'j(j_j`d`dadbZae ej'j(jbj*dcdddeZce ej'j(jdj*dfdgdhZde ej'j(jej*didjdkZfe ej'j(jgjhdldmdnZie ej'j(jjj,dodpdqZke ej'j(jlj*drdsdtZme ej'j(jnj*dudvdwZoe ej'j(jpj*dxdydzZqe ej'j(jrj*d{d|d}Zse ej'j(jtj*d~ddZue ej'j(jvj*dddZwe ej'j(jxj*dddZxe ej'j(jyj*dddZye ej'j(jzj{dddZ|e ej'j(j}j*dddZ~e ej'j(jj*dddZe ej'j(jj*dddZddZe ej'j(jj*d]ddZe ej'j(jjdddZe ej'j(jjhdddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*gdddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jjhdddZe ej'j(jj`dddZe ej'j(jj*de ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj,dddZe ej'j(jj*d]ddZe ej'j(jjdddZe ej'j(jj,dddZe ej'j(jjdddZe ej'j(jj,dddÄZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gd,ddńZe ej'j(jj,dddDŽZe ej'j(jj,dddɄZe ej'j(jj*ddd˄Ze ej'j(jj*ddd̈́Ze ej'j(jj*dddτZe ej'j(jj*dddфZe ej'j(jj*d҃ddԄZe ej'j(jj*duddքZe ej'j(jj*d׃ddلZe ej'j(jj*dڃdd܄Ze ej'j(jj*ej'j(jj*gd,ddބZe ej'j(jj*d,ddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jjېdddZe ej'j(jjېdddZe ej'j(jj`d,dd Ze ej'j(jj*d d d Zdd lmZmZddlmZee  d/dej`dej`dej`de!dededeeefdededeej`ej`ffddZee  d/dej`dej`dej`dej`dej`dej`dej`dee!efd edededeeefdededeej`ej`ej`eeej`d!fffd"d#Ze ej'j(jj*d$d%d&Zejd'd($Z(e(d&ed)e(d&ed*e(d&ed+WddS1swYdS(0N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention NestedTensor__all__JAGGED_OPS_TABLEFcsddlm}t|ttfr$t|fdd|D}t|t|S|r+||}|dkr3|ks5J|dkr=dS|dS)Nrcanonicalize_dimsc3|] }t|VqdSN_outer_to_inner_dim.0dndim ragged_dimI/var/www/auris/lib/python3.10/site-packages/torch/nested/_internal/ops.py z&_outer_to_inner_dim..r)torch._prims_commonr isinstancetuplelisttypedictfromkeys)rdimrZ canonicalizer outputrrrrs  rTc Csvddlm}|||}||kr|st|d|dkr%|s%t|d|r-t|||n|} |r9|dk} | | fS| S)Nrr z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)rr RuntimeErrorr) rr"rop_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr wrappedretoperating_on_batchrrr_wrap_jagged_dim's   r,c sddlmt|ttfsJdt|fdd|D}d|v}|v}tfdd|D}ttfdd|D}||||fS) z For NestedTensor operators, wraps dimensions to non-negative values, and returns metadata related to reduction dimension(s). rr z<_wrap_jagged_dims(): cannot iterate over dimensions of type csg|]}|qSrrr)r rrr Ps z%_wrap_jagged_dims..c3s |] }|dko |kVqdS)rNrr) ragged_idxrrrVz$_wrap_jagged_dims..c3r rrr)rr.rrrZr) rr rrrranyr r!) rdimsr%r.Z wrapped_dimsZoperate_on_batchZoperate_on_raggedZoperate_on_non_batchZouter_to_inner_dimr)r rr.r_wrap_jagged_dimsDs    r2 schema_strreturnc s|d}dd|Dd}t||}|ddkr"|dd}n't||kr0t|t|ksItd|jd|d |d t|d t|d d dddddddd}t|D]k\}} | d\} } | d} | rt| ddn| } | |vrtd| |t|kr| std|jd|d| q\|| | ffdd }|||sdddddd}td|jd|d| d || q\dS)!N, cSsg|]}|dqS)?)endswithrxrrrr-bz check_schema..T... NestedTensor (z): expected at least z arguments and at most z arguments, but got: z argumentscSst|tjo t|t Sr)rtorchTensorrr9rrrqzcheck_schema..cSst|to|jduo|jdkSNr)rr_lengths _ragged_idxrArrrrBrs  cSs t|tSrrrrArrrrBuscSdS)NTrrArrrrBx)tjtjt_allr0z: r6zUnknown arg type: z) missing required argument: cs|r |dup |S|Srr)r9 is_optionalZ _check_fnrrcheck_fnszcheck_schema..check_fnZtensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz )rJzt?rKrLr0z ): expected z to be a ) splitcountlen ValueError__name__ enumerater7keysAssertionError)r3funcargskwargsZnamed_arg_typesZnum_optional_argsZmin_argsZarg_type_check_fnsiZnamed_arg_typenameZarg_typerMZnormalized_arg_typerOZ type_to_descrrNr check_schema`s`        r]aa_namebb_namecCs:|j|j|j|jkrtd|jd|d|ddS)Nr=z : expected  and z' to have the same exact offsets tensor.)_sizerFr$rT)rXr^r_r`rarrrcheck_ragged_dim_sames rdcCsL|jd}|jd|}|d|}t|t|ko%tddt||DS)Nrcss$|] \}}||kp |dkVqdS)r;Nr)rnssrrrr"z%raggedness_matches..)rFrcrRallzip)ntsizeendZ nt_raggedZ size_raggedrrrraggedness_matchess  rmcCsB|dkr|jddkr|d}|dkr|jddks |SNrr)r"shapesqueeze)rJrrrsqueeze_leading_oness rqcs4ttsgttsgfdd}|S)Ncs2D]}fdd}D]}||||<q qS)Ncsfdd}|S)Ncs.tg|Ri|g|Ri|Sr)r])rYrZ)aten_oprXr3rrinnersz@register_func..wrapper..get_inner..innerr)rrrs)rXr3)rrr get_innersz1register_func..wrapper..get_innerr)rXrrrttableaten_opsr3tablesrXrwrappers zregister_func..wrapper)rr)rxrwr3rzrrvr register_funcs  r{c Os$t|d}|dur |Stjj|jvrddlm}|D]}||r,td|j d|qt dd|D}|dkrzg}|j j D]}t |jtjrT||jdn ||jd q@|d d |}t||g|Ri|tt|S|d krtd |g|Ri|tt|SdS)Nr is_nested_intr=z: invalid argument css|] }t|tjVqdSr)rr?r@r8rrrrrz lookup_jagged..rz: jt_allz: anyr<r5zlhs: any, rhs: any, ...)r getr?TagZ pointwisetags%torch.fx.experimental.symbolic_shapesr}r$rTsumZ_schema argumentsrrZ TensorTypeappendr\joinr] functoolspartialjagged_unary_pointwisejagged_binary_pointwise) rXrYrZZ dispatch_funcr}argZnum_tensor_argsZ schema_partsZcheck_schema_strrrr lookup_jaggeds6       rcCs|||j|jd}|S)Noffsetslengths_metadata_cacherFr)rrZrrrextract_kwargss rcs>tdd|Dt|fdd|Di|fitS)Ncs|] }t|tr|VqdSrrGrrrrrrz)jagged_unary_pointwise..c3s"|] }|ur |jn|VqdSr_valuesrnjtrrrs nextrrrXrYrZrrrrs rcsN|d|d}}t|tst|tsJd}t|trMt|trMt||jrAt||j|jg|ddRi|fit|St||j|j|jt|t}|rXt|nt|}|rb||fn||f\}| krtt dt |} | dkr|rj| fn| jf\} } t|| | g|ddRi|fi|S| | kr|j d|j dkrt||j|j |j ddl m} j} j}|jj jd}dur||fddt|j D}| |jj|| |d }|r|fn|f\} } || | g|ddRi|St||j|j |j ) NrrzHcannot call binary pointwise function {} with inputs of shapes {} and {}r~z+NYI: broadcasting NT with T with larger dimnested_from_paddedcs"g|] \}}|jkr |nqSrrF)rr[r9rj padded_max_Srrr-^"z+jagged_binary_pointwise..rr.sum_S min_seqlen max_seqlen)rrrmrcrrr$formatrTr"NotImplementedErrorrqro nested_tensorr_maybe_min_seqlen_maybe_max_seqlenrFexpandrU_offsets)rXrYrZr^r`Zmismatch_error_msgZa_is_ntZextracted_kwargsrJZ t_squeezedlhsrhsrrrtotal_LZt_as_ntrrrr$s^ " ,  rcOs|tjjjurt|i|S|jdkr)||djg|ddRi||dS|jdkrddd}t|||dd \}}|d }t | |d |j dd d }t | |d|j dd d }||krd|St tj|j||d} g|jd|| |j|ddR} |j| S|jdkrddd} t| ||dd \}}|d }|d} | |j d} t| | krtdtj||i|WdS1swYt|)NZapply_rrflattenr;cSdSrr)input start_dimend_dimrrr _flatten_sigz+jagged_torch_function.._flatten_sigTrYrZZnormalize_to_only_use_kwargsrrF)r&rZrms_normcSrrr)rnormalized_shapeweightepsrrr _rms_norm_sigrz,jagged_torch_function.._rms_norm_sigrzNrms_norm(): Normalization over the ragged dim not supported for nested tensors)rr;NN)r?Z_CZ_nnZscaled_dot_product_attentionrrTrrpopr,r"rFrreduceoperatormulroreshaperRrSZDisableTorchFunctionSubclassr)rXrYrZr_ new_kwargsinprrproductZ new_shaperrZmax_normalizablerrrjagged_torch_functionrs\ $   *         rz self: jt_allcOs|tjjjjkr dS|tjjjjkr|djS|tjjjjkr&t|djS|tjjj jtjjj jfvrW|dj durPt t |dj t|djddS|dj S|tjjjjkrd|djS|tjjjjkrs|djSdS)NFrr~)r?opsatenis_non_overlapping_and_densedefaultsym_sizercr"rR sym_numelnumelrEintrmathprodr sym_strideZ_stridessym_storage_offsetZstorage_offsetrrrrtensor_attr_supported_getters  * rcOstjSr)r?Zjaggedrrrrprim_layout_defaultsrcOs|tjjjjkr tddS)NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)r?rrrkrr$rrrrtensor_attr_unsupported_getters rcOspddlm}t|||dd\}}|d}|durdS|dtj|d<|dtjkr/dS||j fi|S)Nr)is_contiguous_for_memory_formatTrrF memory_format) rrrrrrr?contiguous_formatZpreserve_formatr)rXrYrZrrrrrrris_contiguous_generals   rz!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c Ost|||dd\}}|d}t|}|jdur:|dtjkr:ddlm}|jdks.Jd|| dd\}}|St ||j fi|fi|S) NTrrrr)jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r) rrrrEr?rrrrFunbindrr) rXrYrZrrrZnew_metarZcontigrrr clone_defaults    rzinput: jt, weight: t, bias: t?cO@t|||dd\}}|d}t||jfi|fit|SNTrrrrrrrrXrYrZrrrrrrlinear_default#  "rz6self: jt, grad_output: jt, weight: t, output_mask: anycOst|||dd\}}|d}|d}|d}|d}d\} } } t||d|d|d r>tt|j|fit|} |d r^|jd | d } |jd | d } t| | } |d rt t |jj d }|d krv|j} n tj|j|dd} | | | fS)NTrr grad_outputr output_mask)NNNselfrrr;r~rF)keepdim)rrrdrr?matmulrrrrkrJrrangercloner)rXrYrZrrrrrrZdsZdwdbZgrad_2dZinput_2dZ reduce_dimsrrrlinear_backward_default.s2        rzinput: jt_all, dtype: anycOrrrrrrrto_dtypeWrrcOsddlm}t|||dd\}}|d}|d||jfi|}|jj|jd}d} |jdur:|jj|jd} dd l m } dd l m } m } |jdurP|jn|j} | durY|n| }t|| | fro| |}| | }|j|_n|| ||<t|}||d <| |d <t|fi|}|S) Nr_tensor_symint_registryTrrlayoutdevicer FakeTensorFunctionalTensormb_unwrap_functional_tensorrr)rrrrrrtorrEtorch._subclasses.fake_tensorr#torch._subclasses.functional_tensorrrrnested_int_memorr)rXrYrZrrrr new_values new_offsets new_lengthsrrr ragged_source new_thingtgtsrc inp_kwargsr#rrrto_copy_defaultbs2       rz-self: jt_all, src: jt_all, non_blocking: any?c Ost|||dd\}}|d}|d}|j|jkrO|}dd|D}|} dd| D} || kr@td|jd |jt|| D] \} } | | qE|j|j|S) NTrrrcSg|]}|jqSrrorcrrrr-z copy_default..cSrrrrrrrr-rz|jdkr>td |rI|jdurItd |dd |d<|rt j j j t j jj|j|jjd d |jg|jgtd d|jd} t j jj| |jg|jjd djd g|jjddR} t| fit|St||jfi|fit|S)NTrr"zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorrsoftmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorrzesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorrr;z-inf) max_lengthsZ padding_valuer"r)rrrr$rr2r"rFrEr?nn functionalrrr_jagged_to_padded_dense_forwardrrror _max_seqlenfloat_padded_dense_to_jagged_forwardrr) rXrYrZrrrreduce_on_batchreduce_on_raggedZ_reduce_on_non_batchZpadded_softmax_valuesZsoftmax_valuesrrr_softmax_defaultsx     "rz7grad_output: jt, output: jt, dim: any, input_dtype: anycONt|||dd\}}|d}|d}t||j|jfi|fit|S)NTrrr#r)rXrYrZrrgrad_outr#rrr_softmax_backwardLs  rz!self: jt, float: any, train: any?cOs\t|||dd\}}|d}||jfi|\}}t|fit|t|fit|fSrrrrrr)rXrYrZrrrZout1Zout2rrrnative_dropout_default[s r z%grad_output: jt, mask: jt, scale: anycOr)NTrrmaskr)rXrYrZrrrr!rrrnative_dropout_backward_defaultks  r"z2self: jt_all, dim: any, keepdim: any?, dtype: any?cOt|ddg|Ri|S)Nrr_apply_reductionrrrr prod_dim_int{r&zself: jt_all, dtype: any?cO0t|||dd\}}|d}||jfi|Srr rrrr prod_default  r)z$self: jt, split_size: any, dim: any?cs^t|||dd\}}|dt|djd|d<tfdd|jfi|DS)NTrrr"rPc3s&|]}tdd|itVqdS)valuesNrrrr8rrrrs  zsplit_tensor..)rrr,r"rFrrrXrYrZrrrr-r split_tensors  r/z%self: jt, split_sizes: any, dim: any?csZt|||dd\}}|dt|djd|d<fdd|jfi|DS)NTrrr"split_with_sizesc"g|] }tdd|itqSr+rr,r8r-rrr-z,split_with_sizes_default..)rrr,r"rFrr.rr-rsplit_with_sizes_defaults  r4z+self: jt, dim: any, start: any, length: anycOsdt|||dd\}}|d}t||d|jd}||j||d|dd}t|fit|S) NTrrr"narrowstartlengthr"r6r7)rrr,r"rFrrr)rXrYrZrrrr"r+rrrr5s r5z self: jt, chunks: any, dim: any?c st|||dd\}}|dt|djddd\|d<}|rd|d}j}||}dd |D} d d | D} fd d | Dd d |D} j | fd d t dt DSfdd |jfi|DS)NTrrr"chunkr(chunkscSsg|] }tj|ddqS)rr)r?cumsumr8rrrr-sz!chunk_default..cSsg|] }tj|dddqS)rrr)value)Fpadr8rrrr-scsg|]}|jdqS))rrFr)rZ per_offsetsr-rrr-s cSsg|]}|qSr)ritemr8rrrr-rCcs&g|]}tdd|i|qSr2rrr[) chunk_values nested_kwargsrrr-srcr1r2r,r8r-rrr-r3) rrr,r"rFrdiffr9rrPrrR) rXrYrZrrr+r;rZchunked_lengthsZchunked_offsetsZ split_sizesr)rCrrDr chunk_defaults0        rFzself: jt_all, dim: any?c st|||dd\}}|d}|dkrtd|d}||}|}|jddttdt ttffd d } |durU| | t j d d Sdkr]td | | | fddt|jdDS)NTrr"rz2unbind(): only supported for NestedTensor on dim=0rrErcsd}jd}tt|D],}t||t|||k|||7}|dur;t|||||kddqt||k|dur`tt|D]}t||t|||kqMdSdS)NrrcSrH)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionrrrrrrBrIz2unbind_int.._torch_check..)rorrRr?Z_check_is_size_check)rErZ lengths_sumZragged_dim_sizer[)r.r+rr _torch_checks&  z unbind_int.._torch_checkrrzAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)cs*g|]}tjd||dqS)rr8)r?r5rBZlengths_scalarsZoffsets_scalarsr.r+rrr-1szunbind_int..r)rr$rr+rrrFrrrrEtolistr?rPrro) rXrYrZrrr"rrrrHrrIr unbind_ints4  &   rKzself: jt, dim: anycOsbt|||dd\}}|d}|j}tt|j|d|jd|d<t||fi|fit|S)NTrrr"rp) rrrr,rRrcrFrr)rXrYrZrrrr+rrr squeeze_dim<s  rLzself: jt_all, dim: anyc Ost|||dd\}}|d}|j}|d}tt|jd||jddd|d<t|}|d|jdkr<|dd7<t||fi|fi|S) NTrrr"r unsqueeze)r'rF) rrrr,rRrcrFrr) rXrYrZrrrr+r"r rrrunsqueeze_defaultKs  rNztensors: any, dim: anycst|||dd\}}|d}dd|D}t|dksJ|dfdd|D}|d}ttj|jd |d<t|d d|Dfi|fit|dS) NTrtensorscSsg|]}|jr|qSr) is_nestedrrJrrrr-kr:zcat_default..rcs g|] }|jr |n|qSr)rP expand_asrQfirstrrr-ns r"catcSrrrrQrrrr-wr)rrrRr,rorFrr)rXrYrZrrrOnestedr"rrSr cat_defaultbs"  rWzself: any, other: anyc s(t||dd\}}|d}|d}fdd}fdd}|jrl|jsl|d krB||krB|j|d krB|||S|d krk||krk|j|d krkt|j|fi|fit|Sn|js|jr|d kr||kr|jd kr|||S|d kr||kr|jd krt||jfi|fit|SnX|jr|jr|d kr|d krt||j rt|j|jfit|S|d kr|d kr|jd kr|jd kr| |j| |jkrt |||St d |j d |j)NTrrothercs fddt||DS)Ncsg|] \}}||qSrr)rZa_compZb_compryrrr-sz8matmul_default.._unbind_impl..)rir)r^r`ryrr _unbind_impls z$matmul_default.._unbind_implc s|jr|}n|}ddlm}|j}|j}|}|jj|jd}|dur%|}g|jd|j||j|jddR}|jd|d} |jrL| |} n|| } || |j |j|||dS)Nrr output_sizer) rPrrrrrrorFto_padded_tensorr) r^r`rjrrrrr padded_shapeZ padded_ntZpadded_tryrr _padded_impls:   z$matmul_default.._padded_implrr~z1matmul(): not supported between inputs of shapes rb)rrrPr"rFrrrrmrcrkr?stackr$ro) rXrYrZrrrrXrYr_rryrmatmul_default{sV     %    & &$  rbzself: jt_all, mat2: anycOsdt|||dd\}}|d}|d}|dkrtd|dkr(tdttjjjj ||S)NTrrZmat2r`zbmm(): input must be 3Dzbmm(): mat2 must be 3D) rrr"rSrbr?rrrrrXrYrZrrrrXrrr bmm_defaults    rdz'self: jt_all, size: any, implicit: any?cst|||dd\}}|d|dd|vs|drJts.tdjdfdd td D}t|j|fit S) NTrrrkZimplicitzexpand(): cannot expand shape z -> cs"g|] }|jkr dn|qS)r;rrrrkrrr- rz"expand_default..r) rrrmr$rcrr"rrr)rXrYrZrrZ expand_argrrerexpand_defaults  rfzself: t, other: jtcOsDt|||dd\}}|d}|d}t|||jfit|S)NTrrrXrrcrrrexpand_as_default s   rgzself: jt_all, size: anycOslt|||dd\}}|d}|d}t||kr2|gddt|t|D|Std)NTrrrkcss|]}dVqdS)rNr)rrrrrr#szbroadcast_to..zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)rrrRr"rrrS)rXrYrZrrrrkrrr broadcast_tos  ,rhz tensors: anyc Ost|||dd\}}|d}t|dkrtdt|dkr#|dSg}tjdd|D}td d|D}|D],} | jrH|| |q:| t|krc|t | |j j fit|q:td t|S) NTrrOrz7broadcast_tensors(): expected at least one tensor inputrcss|]}|jVqdSrrrQrrrr8sz$broadcast_tensors..csrrrGrQrrrr:rzubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)rrrRrSr?Zbroadcast_shapesrrPrrhr"rrrorr) rXrYrZrrrOZoutsZbroadcast_shaperrJrrrbroadcast_tensors+s,   riz(condition: jt_all, self: any, other: anycOspt|||dd\}}|d}|d}|d}t|||\}}}t||j|j|jfi|fit|S)NTr conditionrrX)rrr?rirrr)rXrYrZrrrjrrXrrr where_selfKs   rkzself: jt, device: any?cOrrrrrrr_pin_memory_default`rrlcOr(rr rrrris_pinned_defaultkr*rmzself: jt_all, other: jt_allcOs|dj|djkSrn)rcrrrris_same_size_defaultvsrncsDt|||dd\}}|d|ddup%t|dttfo%t|ddk}|rU|jfi|}|ddrSt|ttfrMt|fdd |D}|S| j }|S|d} t|dttf} | sg| g} t | |j \} } } }| s|| d} | |d<| rj durt|d dd lm}| r| r|jfi|}|ddr|d d |}|S|rt|d|d| rĈj gnj }||fd|i|S| rt|d|jfi|}t|ddst|dttfr|dn|dg}|D]}|j dkrdd8<q|fdd |S)NTrrr"rrFc3s|] }|jVqdSr)rMrF)ror-rrrrz#_apply_reduction..zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapcSs |dS)Nr)rMrorrrrBs z"_apply_reduction..zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsrrFcst|fiSrrrq) out_kwargsrrrBs)rrrrrrRrrrrMrFr2r"rEr$Ztorch.utils._pytreerpr]r)rX func_nameZidentity_elementrYrZrrZfull_reductionoutZdim_to_convertZ is_dimlistZ converted_dimrrreduce_on_non_batchrpZ dim_to_passZdimlistrr)rrrrr%}s         r%cOr(rr rrrr sum_defaultr*rvz3self: jt_all, dim: any?, keepdim: any?, dtype: any?cOr#)Nrrr$rrrrsum_dim_IntListr'rwz"self: jt_all, dim0: any, dim1: anyc Os.t|||dd\}}ddlm}|d}|||d|df\}}||jks.||jkrj|dks6|dkr:td||jkrB|} n|} t|} | | d <t| t t |j ||jt t |j ||jfi| St||d|jd |d<t||d|jd |d<t||jfi|fit|S) NTrrr rdim0dim1z?Transpose is not supported on the batch dimension for jagged NTrF transpose)rrr rr"rFrSrrr+rzrrRrcr,r) rXrYrZrrr rrxryZto_dimrrrr transpose_ints>   "r{zself: jt_all, dims: anyc st|||dd\}}|d|d}t}tjt|kr3tdddt|dd d lm}||}t|tt|krLtd j durUtd |d d kr_td | j |d<fdd|ddD} | |d<t |j fi|fi|S)NTrrr1z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().rr z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrFcsg|] }t|jqSr)rrF)rr"rZinp_dimrrr-Hs z#permute_default..r)rrrrRrcrSrr setrEindexrFrr) rXrYrZrrr1rr Zcanonicalized_dimsZ inner_dimsrr}rpermute_default's>           rcst|||dd\}}|d|djdkr.tjtkr.tdjddtd ks9tsDtd jd fd d fddttdD}t  t |j |fitWdS1szwYdS)NTrrrkrzVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r|r`zview(): cannot view shape z as cs&|jdkr j|S|dSrD)rFrrk)Z inner_idxrerrget_inner_sizets  z$view_default..get_inner_sizecsg|]}|qSrrrB)rrrr-{sz view_default..)rrrFrrcr$rRrmrr?Zinference_modeZ is_inferencerrr)rXrYrZrrZ inner_sizer)rrrkr view_defaultPs(  $rzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anycOst|||dd\}}|d}|dkrtd|d}|j|j}|t|}|dkr3td||vr@|jdur@td ||vrtj j j |j j |jd |jg|jgd } tj j j tj|j jdd f|j|jd |jg|jgd | j} |jd d | jd} tj| ddd| } | | | } tjt| ddd| }t||d}| |}tj j j||jg|j jddd|j|jd d}t|fit|| |fS||j fi|\}} }t|fit|| |fS)NTrrr~zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsrrzVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr)rr)rdtype)rr~)r"rrrr;)rrr"r$rorFrRrEr?rrrrrrrZonesrrrrErMrZsquaresqrtr unflattenrr)rXrYrZrrrrZ ragged_sizeZnum_dims_not_normalizedZ padded_inputZ padded_maskZragged_lengthsmeanZpadded_normalizedZvarianceZstdZpadded_layer_normZjagged_layer_norm_valuesr#rrrnative_layer_norm_defaults       rzpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc Ospt|||dd\}}|d}|d}||j|jfi|\}}} |dur+d|| fSt|fit||| fS)NTrrrr) rXrYrZrrrrZd_inputZd_gammaZd_betarrr"native_layer_norm_backward_defaults   rz"self: jt_all, dim: any, index: anycOst|||dd\}}|d}t||d|jddd\|d<}|r+||dS|jdur4tdt|}|d|jd krI|d d 8<t ||j fi|fi|S) NTrrr"selectr:rzSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesrrF) rrr,r"rFrrErSrrr)rXrYrZrrrr+rrrrr select_ints"  rz7self: jt, dim: any?, start: any?, end: any?, step: any?cOs\t|||dd\}}|d}t||d|jd|d<t||jfi|fit|S)NTrrr"slice)rrr,r"rFrrrrrrr slice_tensors "rz8input: jt_all, indices: any, values: t, accumulate: any?cOst|||dd\}}|d}|d}t||ksJt||jdkr|s/tdddlm}|j }|j } | } |j j |jd} | durL| } g|j d|j| |j |jddR} |j d| d } ||| |fi||j|j| || d }|tjjjjkr|j ||S|S|dur|}n|}tt||j|kd ||d ||j}|d|j|g||jdd}|tjjjjkr||j |fi||_ |St||j |fi|fit|S) NTrrindicesrz[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsrrZr[rz7Some indices in the ragged dimension are out of bounds!r)rrrRr"rF is_contiguousr$rrrrrror]rr?rr index_put_rr r+rrrEZ _assert_asyncrhrr)rXrYrZrrrrrrrrrr^Z padded_inpZnew_njtrZragged_indicesZ func_indicesrrrr'sx        rz{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anycOrrrrrrrconvolution_defaults  "rcOst|||dd\}}|d}t||dd|j\}}}}|rg|sg|r&J|d} d|d<ttjjjj ddfi|} |j durD|j n|j } t | dD]}| d } qQ| | } | se| |j} | Sd } t|d| fi|S) NTrrr"rrrrr;gzG?)rr2r"rFr%r?rrr dim_IntListrErrErrMrp)rXrYrZrrrrrrurZintermediate_sumrrtZintermediate_valuerrrmean_dims8    rcOr(rr rrrr mean_defaultr*rz&self: jt_all, dim: any?, keepdim: any?cOr#)Nr0Fr$rrrrany_dimsrz%self: jt_all, dim: any, keepdim: any?cO:t|||dd\}}|dg|d<ttjjjjfi|SNTrr")rrr?rrr0r1r.rrrany_dim  rcOr#)NrhTr$rrrrall_dimsrrcOrr)rrr?rrrhr1r.rrrall_dimrrcOr(rr rrrrall_any_max_min_defaults   rcO@t|||dd\}}t|djj}t|d|g|Ri|S)NTrrminrr?finformaxr%rXrYrZrrZ dtype_maxrrrmin_dim  rcOr)NTrrrrr?rrrr%rXrYrZrrZ dtype_minrrrmax_dimrrcOr)NTrraminrrrrr amin_default   rcOr)NTrramaxrrrrr amax_defaultrrcOr)NTrrargminrrrrrargmin_default%rrcOr)NTrrargmaxrrrrrargmax_default1rrzAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc sddlmt|||dd\}}|d}|j|d<|d}|j|d<tfddt|d D}tt|d |d |d |d <|d }|j |j d ||<|d d}||d <t |} || d <t |di|fi| S)Nrr|Trgradrc3s |] \}}|r|VqdSrr)rr[rfr|rrrMr/z=value_selecting_reduction_backward_default..sizesr""value_selecting_reduction_backwardrrFr) rr}rrrrrUr,rRrkrFrr) rXrYrZrrrrr.rr rr|r*value_selecting_reduction_backward_default=s,        rcOst|||dd\}}|d}|D]%}t|tstd||dkr*tdt||djs6tdqt|dd|d |dj d |d <t|d d |Dfi|fit |dS) NTrrOz+stack(): expected all nested tensors inputsrz9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurerr"racSrrrrQrrrr-|rz!stack_default..) rrrrr$r"rmror,rFr)rXrYrZrrrOrJrrr stack_defaultas0   rzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?cOsLt|||dd\}}|d}|d}t|||jfi|fit|S)NTrrrr)rXrYrZrrrrrrrembedding_defaults  rzYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anycOs>t|||dd\}}|d}|d}||j|jfi|S)NTrrrr )rXrYrZrrrrrrr embedding_dense_backward_defaults   rcOs(t|||dd\}}|d}|jSr)rrrdetachrrrrvalues_defaults   rcOs(t|||dd\}}|d}||jSrr rrrr all_defaults   rz-self: jt_all, padding: any, output_size: any?c Oslt|||dd\}}|d}|jdurtd|d}|dur&||j}n|jdur.|jn|jd}| }|jdkrF| |jdd}|j } | dkrV|j dd }n | dkra|d }|jtju} | rr|jrr|tj}tjj||jg|g|d } | r| jr| tj} t| dkr| d | dd} n t| dkr| d } |jdkr| |jd} | S) NTrrz?to_padded_tensor(): not supported for nested tensors with holesr\rrr~rr;padding)rrrEr$rFZ_max_seqlen_tensorrrrkr+rzror"rrMrr?boolis_cudarhalfrrrrrRrrp) rXrYrZrrrr\Z max_seq_lenr+Z values_shapeis_boolZ padded_outrrrto_padded_tensor_defaultsR                  rzcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?cOs`t|||dd\}}|d|d}}|dd}|dkr#||d}|j}|dkr3|jdd }n |dkr>|d }|jtj u} | rO|j rO| tj }tj j||g|d } | rf| j rf| tj } t|dkrw| d |dd} n t|dkr| d } |dkr| |dd } |d } |d} i} | dur| | d <| dur| | d<t| ||| dS)NTrpaddedrr.rr`r~rr;rrrr)rFr)rrrzror"rrMrr?rrrrrrrrRrrpr)rXrYrZrrrrr.Zpadded_ragged_dim1_shaperr+rrmetadata_cacherrr"_nested_from_padded_tensor_defaultsN               rzcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c Os~t|||dd\}}|d|d|d}}}|d}|d} |d} i} | dur.| | d<| dur6| | d<t||||| d S) NTrrrrr.rr)rrFr)rr) rXrYrZrrr+rrr.rrrrrr _nested_view_from_jagged_default6 s,  rcO$t|||dd\}}|d}|jSr)rrrrrrr_nested_get_offsetsV  rcOrr)rrrErrrr_nested_get_lengths` rrcOrr)rrrFrrrr_nested_get_ragged_idxj rrcO,t|||dd\}}|d}|jddS)NTrrrrrrrrrrr_nested_get_min_seqlent  rcOr)NTrrrrrrrr_nested_get_max_seqlen~ rrzself: jt, mask: anyc Ost|||dd\}}|d}|d}|jdkrtd|j|jkr/td|jd|j|j|}t |j d d d }t |}||j |d <t dd |i|S)NTrrr!r~z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape rrr=rr+r)rrrr$ror masked_selectr+r?r@r<rrr) rXrYrZrrrr!Z res_valuesZ mask_cumsumrrrmasked_select_default s(    rz2grad_output: t, self: jt_all, dim: any, index: anycOsVt|||dd\}}|d}|d}tj||jd}||d|d||S)NTrrr)rr"r)rrr? zeros_likerrr )rXrYrZrrrrZ grad_inputrrr_nested_select_backward_default s  rzself: jt_all, s: anycOsF|d}|d}||j|||j||jdur!||j|dSdSrn)rrrE)rXrYrZrstreamrrrrecord_stream_default s   rzRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?cOsHt|||dd\}}|d}t|ddkr ||jfi|Std)NTrrrkrz2new_empty() not supported for NJT with shape != ())rrrRrr$rrrrnew_empty_default s  rzself: jt_all, ...cOs:tdd|D}t|dd|Di|fit|S)NcsrrrGrrrrr rz&activation_backward..css$|] }t|tr |jn|VqdSr)rrrrrrrr rgr)rXrYrZrrrractivation_backward s  rzself: jt_all, value: anycOrrrrrrr fill_Scalar rrcOs4t|||dd\}}|d}||jfi||Srr rrrr fill__Scalar s  rc OsTt|||dd\}}|d}t|}||j\}}t|fi|t|fi|fSr)rrrrr) rXrYrZrrrr Zmantissaexponentrrr frexp_Tensor s rz+grad: any, self: any, other: any, mask: anyc Ost|||dd\}}|d}|d}|d}|d}|dur$dSd} |dr4t||d d } d} |d rDt|d d |} | | fS) NTrrrrXr!rrr;r)rrr?rrz) rXrYrZrrrrrXZgrad_input_maskZ grad_selfZ grad_otherrrrmatmul_backward_default s     r)flex_attentionflex_attention_backward) GraphModulerquerykeyr> score_mod block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc Cs|dkr|dkr|dksJtdd||Dr#tdt|d|d|d||||||d } tjj| d dd d|j |j |j |jd  dd} tjj| d dd d|j |j |j |jd  dd} | | fS) Ncss"|] }t|tjo |jVqdSr)rr?r@rP)rbufrrrrC s  zflex_njt..zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r)rrrrrrrr~rr)r"r0r$flex_attention_hopr+rMr?rVnested_tensor_from_jaggedrzrprrErr) rrr>rrrrrrr#Z output_njtZ logsumexp_njtrrrflex_njt4 sJ(     rrt logsumexprgrad_logsumexpfw_graph joint_graph.cCst|d|d|d|d|d|d|d||| | | | | d}|\}}}}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}||||fS)Nr) rtrrrrrrrrrrrr~r) flex_attention_backward_hopr+rMr?rVrrzrprrErr)rrr>rtrrrrrrrrrrr#Z dense_q_gradZ dense_k_gradZ dense_v_gradZscore_mod_other_buffer_gradsZ njt_q_gradZ njt_k_gradZ njt_v_gradrrrflex_njt_backwardm sZ         rz self: anycOsddlm}|S)Nr)_nt_view_dummy)Z$torch.nested._internal.nested_tensorr)rXrYrZrrrr_nested_get_jagged_dummy s rrZIMPLZCPUCUDAZMeta)F)TFF)r)rr)rrrtypingrr?Ztorch.nn.functionalrrr?Ztorch.fx.operator_schemasrZtorch.nested._internal.sdparrrr rAny__annotations__r Dictrr,r2strr]rdrmrqr{rZregister_jagged_funcCallablerrrrrrrrrrr"rrrrrZprimrrrkrrrrrrZlinearrZlinear_backwardrrrrZ_to_copyrr r rZ empty_likeZ ones_likerZ rand_likeZ randn_liker Z full_likeZ randint_likeZ low_dtypeZzero_rZ_softmaxrZ_softmax_backward_datarZnative_dropoutr Znative_dropout_backwardr"rZdim_intr&r)rPr@r/r0r4r5r9rFrrrKrprLrMrNrUrWrrbZbmmrdrrfrRrgrhriwhererrkZ _pin_memoryrl is_pinnedrmZ is_same_sizernr%rrvrrwrzr{ZpermuterviewZ _unsafe_viewrZnative_layer_normrZnative_layer_norm_backwardrrrrrZ index_putrZ convolutionrrrrr0r1rrrhrrrrrrrrrrrrrrrrrrarZ embeddingrZembedding_dense_backwardrr+Z_nested_get_valuesrrr]rZ_nested_from_padded_tensorrZ_nested_view_from_jaggedrrrrrrrrZ_nested_select_backwardrZ record_streamrZ new_emptyZ new_zerosZnew_onesrZ elu_backwardZhardshrink_backwardZhardsigmoid_backwardZhardtanh_backwardZsoftplus_backwardZsoftshrink_backwardrfillZScalarrZfill_rfrexprZmatmul_backwardrZ&torch._higher_order_ops.flex_attentionrrrrZtorch.fx.graph_modulerZpy_implTuplerrUnionrrZlibraryZ_scoped_libraryimplrrrrsf       @    )  NH                   %  (        /    H                ) H    m            j     * ( /  a        T    &                                 ;  5                              8       ?$