
    JThR}             &       3   % S SK r S SKrS SKrS SK7  S SKJr  S SKrS SKJs  Jr	  S SK
Jr  S SKJr  SSKJr  / r\\   \S'   0 r\\\4   \S	'   SS
 jr   SS jrSS jrS\SS4S jrS\S\S\S\SS4
S jrS rS rS r\ R>                  " \\5      r S\\!   4S jr"S r#S r$S r%S r&\ " \RN                  RP                  RR                  RT                  \RN                  RP                  RV                  RT                  \RN                  RP                  RX                  RT                  \RN                  RP                  RZ                  RT                  \RN                  RP                  R\                  RT                  \RN                  RP                  R^                  RT                  \RN                  RP                  R`                  RT                  /S5      S 5       r1\ " \RN                  Rd                  Rf                  RT                  S5      S 5       r4\ " \RN                  RP                  Rj                  RT                  /S5      S  5       r6\ " \RN                  RP                  Rn                  RT                  S5      S! 5       r8\ " \RN                  RP                  Rn                  Rr                  S"5      " \85        \ " \RN                  RP                  Rt                  RT                  S#5      S$ 5       r;\ " \RN                  RP                  Rx                  RT                  S%5      S& 5       r=\ " \RN                  RP                  R|                  RT                  S'5      S( 5       r?\ " \RN                  RP                  R                  R                  S)5      S* 5       rB\ " \RN                  RP                  R                  RT                  S5      S+ 5       rD\ " \RN                  RP                  R                  RT                  S,5      S- 5       rF\ " \RN                  RP                  R                  RT                  S5      " \$5        \ " \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  \RN                  RP                  R                  RT                  /S5      S. 5       rM\ " \RN                  RP                  R                  RT                  S/5      " \M5        \ " \RN                  RP                  R                  RT                  S05      " \M5        \ " \RN                  RP                  R                  R                  S15      " \M5        \ " \RN                  RP                  R                  RT                  S5      S2 5       rR\ " \RN                  RP                  R                  RT                  S35      S4 5       rT\ " \RN                  RP                  R                  RT                  S55      S6 5       rV\ " \RN                  RP                  R                  RT                  S75      S8 5       rX\ " \RN                  RP                  R                  RT                  S95      S: 5       rZ\ " \RN                  RP                  R                  R                  S;5      S< 5       r]\ " \RN                  RP                  R                  RT                  S=5      S> 5       r^\ " \RN                  RP                  R                  R                  S?5      S@ 5       ra\ " \RN                  RP                  R                  RT                  SA5      SB 5       rc\ " \RN                  RP                  R                  RT                  SC5      SD 5       rd\ " \RN                  RP                  R                  RT                  SE5      SF 5       rf\ " \RN                  RP                  R                  R                  SG5      SH 5       ri\ " \RN                  RP                  R                  RX                  SI5      SJ 5       rk\ " \RN                  RP                  R                  RT                  SK5      SL 5       rm\ " \RN                  RP                  R                  RT                  SM5      SN 5       ro\ " \RN                  RP                  R                  RT                  SO5      SP 5       rq\ " \RN                  RP                  R                  RT                  SQ5      SR 5       rs\ " \RN                  RP                  R                  RT                  SS5      ST 5       ru\ " \RN                  RP                  R                  RT                  SU5      SV 5       rw\ " \RN                  RP                  R                  RT                  SW5      SX 5       rx\ " \RN                  RP                  R                  RT                  SY5      SZ 5       ry\ " \RN                  RP                  R                  R                  S[5      S\ 5       r|\ " \RN                  RP                  R                  RT                  S]5      S^ 5       r~\ " \RN                  RP                  R                  RT                  S]5      S_ 5       r\ " \RN                  RP                  GR                  RT                  S`5      Sa 5       rSb r\ " \RN                  RP                  GR                  RT                  S=5      Sc 5       r\ " \RN                  RP                  GR                  GR                  Sd5      Se 5       r\ " \RN                  RP                  GR                  R                  Sf5      Sg 5       r\ " \RN                  RP                  GR                  RT                  Sh5      Si 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /SW5      Sj 5       r\ " \RN                  RP                  GR                  RT                  Sk5      Sl 5       r\ " \RN                  RP                  GR"                  RT                  Sm5      Sn 5       r\ " \RN                  RP                  GR&                  R                  So5      Sp 5       r\ " \RN                  RP                  GR*                  R                  Sq5      Sr 5       r\ " \RN                  RP                  GR.                  RT                  Ss5      \ " \RN                  RP                  GR0                  RT                  Ss5      St 5       5       r\ " \RN                  RP                  GR2                  RT                  Su5      Sv 5       r\ " \RN                  RP                  GR6                  RX                  Sd5      Sw 5       r\ " \RN                  RP                  GR6                  RT                  S=5      Sx 5       r\ " \RN                  RP                  GR<                  GR>                  Sy5      Sz 5       r\ " \RN                  RP                  GR<                  RX                  S{5      S| 5       r\ " \RN                  RP                  GRD                  GR>                  Sy5      S} 5       r\ " \RN                  RP                  GRD                  RX                  S{5      S~ 5       r\ " \RN                  RP                  GRD                  RT                  \RN                  RP                  GR<                  RT                  \RN                  RP                  GRJ                  RT                  \RN                  RP                  GRL                  RT                  /S5      S 5       r\ " \RN                  RP                  GRL                  RX                  S{5      S 5       r\ " \RN                  RP                  GRJ                  RX                  S{5      S 5       r\ " \RN                  RP                  GRT                  RT                  Sy5      S 5       r\ " \RN                  RP                  GRX                  RT                  Sy5      S 5       r\ " \RN                  RP                  GR\                  RT                  Sy5      S 5       r\ " \RN                  RP                  GR`                  RT                  Sy5      S 5       r\ " \RN                  RP                  GRd                  RT                  S5      S 5       r\ " \RN                  RP                  GRh                  RT                  SM5      S 5       r\ " \RN                  RP                  GRl                  RT                  S5      S 5       r\ " \RN                  RP                  GRp                  RT                  S5      S 5       r\ " \RN                  RP                  GRt                  RT                  \RN                  RP                  GRv                  RT                  /S5      S 5       r\ " \RN                  RP                  GRD                  RT                  S5      S 5       r\ " \RN                  RP                  GR|                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /S5      S 5       r\ " \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  \RN                  RP                  GR                  RT                  /S5      S 5       r\ " \RN                  RP                  GR                  GR                  S5      S 5       r\ " \RN                  RP                  GR                  GR                  S5      S 5       r\ " \RN                  RP                  GR                  R                  S5      S 5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       rS SKJrJr  S SKJr  \GR                  " \5        SS\R                  S\R                  S\R                  S\!S\S\S\\\4   S\S\S\\R                  \R                  4   4S jj5       r\GR                  " \5        SS\R                  S\R                  S\R                  S\R                  S\R                  S\R                  S\R                  S\\!\4   S\S\S\S\\\4   S\S\S\\R                  \R                  \R                  \\\R                     S4   4   4S jj5       r\ " \RN                  RP                  GR                  RT                  S5      S 5       r\GR                  GR                  SS5       r(\(GR                  S\S5        \(GR                  S\S5        \(GR                  S\S5        SSS5        g! , (       d  f       g= f)    N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention   NestedTensor__all__JAGGED_OPS_TABLEc                 $  ^ ^ SSK Jn  [        U[        [        45      (       a@  [        U5      " U U4S jU 5       5      n[        U5      " [        R                  U5      5      $ U(       a	  U" T U5      nUS:  a  UT :  d   eUS:X  a  TS-
  $ US-
  $ )Nr   canonicalize_dimsc              3   >   >#    U  H  n[        TUT5      v   M     g 7fN_outer_to_inner_dim).0dndim
ragged_dims     R/var/www/auris/envauris/lib/python3.13/site-packages/torch/nested/_internal/ops.py	<genexpr>&_outer_to_inner_dim.<locals>.<genexpr>   s     QS.tQ
CCS   r   )torch._prims_commonr   
isinstancetuplelisttypedictfromkeys)r   dimr   canonicalizer   outputs   ` `   r   r   r      s    5#t}%%cQSQQF|DMM&122c*!8d
"" !AX:>2372    c                     SSK Jn  U" X5      nX:X  a  U(       d  [        U S35      eUS:X  a  U(       d  [        U S35      eU(       a  [        XU5      OUn	U(       a  US:H  n
X4$ U	$ )Nr   r   z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)r   r   RuntimeErrorr   )r   r"   r   op_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr   wrappedretoperating_on_batchs              r   _wrap_jagged_dimr/   '   s     6*G%5gY&VWXX	AogY&QRSS   	D:6 
  %\((Jr%   c                 @  ^ ^ SSK Jn  [        U[        [        45      (       d   S[        U5       35       eU Vs/ s H  oT" T U5      PM     nnSU;   nTU;   n[        U4S jU 5       5      n	[        [        R                  U U4S jU 5       5      5      n
XX4$ s  snf )z~
For NestedTensor operators,
wraps dimensions to non-negative values,
and returns metadata related to reduction dimension(s).
r   r   z<_wrap_jagged_dims(): cannot iterate over dimensions of type c              3   D   >#    U  H  oS :g  =(       a    UT:g  v   M     g7f)r   N )r   r   
ragged_idxs     r   r   $_wrap_jagged_dims.<locals>.<genexpr>V   s     P<aAv9!z/9<s    c              3   >   >#    U  H  n[        TUT5      v   M     g 7fr   r   )r   r   r   r3   s     r   r   r4   Z   s     U1)$:>>r   )	r   r   r   r   r   r   anyr    r!   )r   dimsr(   r3   r   r   wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dims   `  `       r   _wrap_jagged_dimsr=   D   s     6udm  S	Ed4j\RS 
 -1,0q$"D   L("l2P<PP UUU 1BXXs   B
schema_strreturnc                 h  ^ U R                  S5      nU Vs/ s H  oUR                  S5      PM     snR                  S5      n[        U5      U-
  nUS   S:X  a  US S nO^[        U5      U:  a  [        U5      [        U5      ::  d7  [	        SUR
                   SU  SU S	[        U5       S
[        U5       S35      eS S S S S.n[        U5       H  u  pU
R                  S5      u  pUR                  S5      nU(       a  US S OUnXR                  5       ;  a  [        SU 35      eU	[        U5      :  a'  U(       d  [	        SUR
                   SU  SU 35      eM  X   mU4U4S jjnU" X)   5      (       a  M  SSSSSS.n[	        SUR
                   SU  SU SUU    35      e   g s  snf )N, ?T...NestedTensor (z): expected at least z arguments and at most z arguments, but got: z
 argumentsc                 n    [        U [        R                  5      =(       a    [        U [        5      (       + $ r   )r   torchTensorr	   xs    r   <lambda>check_schema.<locals>.<lambda>q   s!    z!U\\2V:a;V7VVr%   c                 x    [        U [        5      =(       a$    U R                  S L =(       a    U R                  S:H  $ Nr   )r   r	   _lengths_ragged_idxrJ   s    r   rL   rM   r   s2    
1l3 JJ$MMQr%   c                 "    [        U [        5      $ r   r   r	   rJ   s    r   rL   rM   u   s    J|
r%   c                     g)NTr2   rJ   s    r   rL   rM   x   s    r%   )tjtjt_allr6   z: zUnknown arg type: z) missing required argument: c                 H   > U(       a  U S L =(       d    T" U 5      $ T" U 5      $ r   r2   )rK   is_optional	_check_fns     r   check_fncheck_schema.<locals>.check_fn   s#    Dy0IaL0 |#r%   tensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz
<any type>)rU   zt?rV   rW   r6   z): expected z	 to be a )	splitendswithcountlen
ValueError__name__	enumeratekeysAssertionError)r>   funcargskwargsnamed_arg_typesrK   num_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typerY   normalized_arg_typer[   type_to_descrZ   s                    @r   check_schemart   `   s    &&t,O2AB/QC/BHHN?#&77H re#)#2.D	X%#d)s?7K*Ka
|;PQYPZ [)),_)=(>>St9+Z)  W
 	 '7'--d3'',/:hsm&=&=&?? #56I5J!KLLD	> #DMM?!J< @2269  &;	$/ 	$   '=6#L a
|<vY)*, A 81 Cs   F/aa_namebb_namec           	          UR                   UR                     UR                   UR                     :w  a  [        SU R                   SU SU S35      eg )NrE   z: expected  and z' to have the same exact offsets tensor.)_sizerQ   r'   rc   )rg   ru   rv   rw   rx   s        r   check_ragged_dim_samer|      sZ     	wwq}}!77DMM?+fXU6( K) )
 	
 8r%   c                     U R                   S-   nU R                  S U nUS U n[        U5      [        U5      :H  =(       a    [        S [	        X45       5       5      $ )Nr   c              3   F   #    U  H  u  pX:H  =(       d    US :H  v   M     g7f)rC   Nr2   )r   nsss      r   r   %raggedness_matches.<locals>.<genexpr>   s"     H,G52BGqBw,Gs   !)rQ   r{   ra   allzip)ntsizeend	nt_raggedsize_raggeds        r   raggedness_matchesr      sX    
..1
C#It*Ky>S-- HC	,GHHr%   c                     U R                  5       S:  aM  U R                  S   S:X  a:  U R                  S5      n U R                  5       S:  a  U R                  S   S:X  a  M:  U $ Nr   r   )r"   shapesqueeze)rU   s    r   squeeze_leading_onesr      sN    & %%'A+!''!*/IIaL %%'A+!''!*/Hr%   c                 |   ^ ^^ [        T[        5      (       d  T/m[        T [        5      (       d  T /m UUU 4S jnU$ )Nc                 P   >^  T H  nU U4S jnT H  nU" U5      X1'   M     M     T $ )Nc                    >^  U UU4S jnU$ )Nc                  <   > [        TT/U Q70 UD6  T" T/U Q70 UD6$ r   )rt   )rh   ri   aten_oprg   r>   s     r   inner@register_func.<locals>.wrapper.<locals>.get_inner.<locals>.inner   s,     TCDCFC9$9&99r%   r2   )r   r   rg   r>   s   ` r   	get_inner1register_func.<locals>.wrapper.<locals>.get_inner   s    : r%   r2   )rg   r   r   tableaten_opsr>   tabless   `   r   wrapperregister_func.<locals>.wrapper   s1    G  !*7!3     r%   )r   r   )r   r   r>   r   s   ``` r   register_funcr      s7    h%%:fd## Nr%   c                 0   [         R                  U S 5      nUb  U$ [        R                  R                  U R
                  ;   GaR  SSKJn  U H+  nU" U5      (       d  M  [        SU R                   SU 35      e   [        S U 5       5      nUS:X  a  / nU R                  R                   Hj  n[        UR                  [        R                  5      (       a   UR!                  UR"                   S35          O"UR!                  UR"                   S35        Ml     UR!                  S	5        S
R%                  U5      n['        X/UQ70 UD6  [(        R*                  " [,        U 5      $ US:X  a+  ['        SU /UQ70 UD6  [(        R*                  " [.        U 5      $ g )Nr   is_nested_intrE   z: invalid argument c              3   V   #    U  H  n[        U[        R                  5      v   M!     g 7fr   )r   rH   rI   )r   rK   s     r   r    lookup_jagged.<locals>.<genexpr>   s     H4ajELL994s   ')r   z: jt_allz: anyrD   rA      zlhs: any, rhs: any, ...)r   getrH   Tag	pointwisetags%torch.fx.experimental.symbolic_shapesr   r'   rc   sum_schema	argumentsr   r   
TensorTypeappendrp   joinrt   	functoolspartialjagged_unary_pointwisejagged_binary_pointwise)	rg   rh   ri   dispatch_funcr   argnum_tensor_argsschema_partscheck_schema_strs	            r   lookup_jaggedr      so   $((t4M  yydii'G
 CS!!"#DMM?2EcUK   H4HHa L||--chh(8(899 ''388*H(=> ''388*E(:; . &#yy6)A$A&A$$%;TBB!2DJ4J6J$$%<dCCr%   c                 t    U R                  5       U R                  5       U R                  U R                  S.nU$ )Noffsetslengths_metadata_cacherQ   r   )r   ri   s     r   extract_kwargsr     s2    ;;=;;=..	F Mr%   c                 r   ^ [        S U 5       5      m[        U " U4S jU 5       0 UD640 [        T5      D6$ )Nc              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rS   r   r   s     r   r   )jagged_unary_pointwise.<locals>.<genexpr>  s     Ddsjl&Cssd   (	(c              3   H   >#    U  H  oTL a  UR                   OUv   M     g 7fr   )_values)r   r   njts     r   r   r     s     BTcSjs{{c1Ts   "nextr	   r   )rg   rh   ri   r   s      @r   r   r     s@    
DdD
DCBTBMfM

 r%   c           	         US   US   pC[        U[        5      (       d  [        U[        5      (       d   eSn[        U[        5      (       a  [        U[        5      (       a  [        X4R                  5      (       a7  [        U " UR                  UR                  /USS  Q70 UD640 [        U5      D6$ [        UR                  U R                  UR                  UR                  5      5      e[        U[        5      nU(       a  [        U5      O
[        U5      nU(       a  X44OXC4u  pU	R                  5       UR                  5       :  a  [        S5      e[        U	5      n
UR                  5       U
R                  5       S-   :  a=  U(       a  UR                  U
4OXR                  4u  p[        U " X/USS  Q70 UD640 UD6$ UR                  5       UR                  5       :X  Ga0  UR                  S   UR                  S   :w  a:  [        UR                  U R                  UR                  UR                  5      5      eSSKJn  UR                  nUR                   nUnUR                  R                  UR"                  S-
     nUc  UnU	R%                  ['        U	R                  5       VVs/ s H  u  nnUUR"                  :w  a  UOUPM     snn5      n	U" U	UR(                  UR"                  UUUS9nU(       a  UU4OUU4u  pU " X/USS  Q70 UD6$ [        UR                  U R                  UR                  UR                  5      5      es  snnf )Nr   r   zHcannot call binary pointwise function {} with inputs of shapes {} and {}r   z+NYI: broadcasting NT with T with larger dimnested_from_paddedr   r3   sum_S
min_seqlen
max_seqlen)r   r	   r   r{   r   r   r'   formatrc   r"   NotImplementedErrorr   r   nested_tensorr   _maybe_min_seqlen_maybe_max_seqlenrQ   expandrd   _offsets)rg   rh   ri   ru   rw   mismatch_error_msga_is_ntextracted_kwargsr   rU   
t_squeezedlhsrhsr   r   r   padded_max_Stotal_Lrn   rK   t_as_nts                        r   r   r   $  s   7DGqa&&*Q*E*EEE 	S  !\""z!\'B'B a))QYY		?DH??CQRSCT  -44T]]AGGQWWUVVL)G,3~a(9J QFA6EBuuw!"OPP%a(J	vvx:>>#a''/6BJJ
+Z<TD?DH??TCSTT 	uuw!%%' 771:#"))$--!''J  	6 ))
))
!**""2>>A#56"L HHDMaggDVWDVDAq!r~~%Q<7DVW
 %KK~~!!
 %,B='2C2tABx2622 )00Q
RR# Xs   .!M
c                 B   U [         R                  R                  R                  L a  [	        U0 UD6$ U R
                  S:X  a!  U " US   R                  /USS  Q70 UD6  US   $ U R
                  S:X  a  SS jn[        X1USS9u  pEUR                  S5      n[        UR                  5       US	   UR                  SS
S9n[        UR                  5       US   UR                  SS
S9nXx:X  a  U$ [        R                  " [        R                  UR                   XxS-    5      n	/ UR                   S U QU	PUR                   US-   S  Q7n
UR"                  " U
6 $ U R
                  S:X  a  SS jn[        XUSS9u  pEUR                  S5      nUR                  S5      nUR                  5       UR                  -
  S-
  n[%        U5      U:  a  ['        S5      e[         R                  R)                  5          U " U0 UD6sS S S 5        $ [+        U 5      e! , (       d  f       N= f)Napply_r   r   flattenc                     g r   r2   )input	start_dimend_dims      r   _flatten_sig+jagged_torch_function.<locals>._flatten_sig      r%   Trh   ri   normalize_to_only_use_kwargsr   r   F)r)   r   rms_normc                     g r   r2   )r   normalized_shapeweightepss       r   _rms_norm_sig,jagged_torch_function.<locals>._rms_norm_sig  r   r%   r   zNrms_norm(): Normalization over the ragged dim not supported for nested tensors)r   rC   NN)rH   _C_nnscaled_dot_product_attentionr   rc   r   r   popr/   r"   rQ   r   reduceoperatormulr   reshapera   rb   DisableTorchFunctionSubclassr   )rg   rh   ri   r   _
new_kwargsinpr   r   product	new_shaper   r   max_normalizables                 r   jagged_torch_functionr  r  s    uxx||8882DCFCC}} T!W__2tABx262Aw }}	!	 +FQU
 nnW% %GGI{#OO!&
	 #GGIy!OO!&
 J""8<<9QR{1STPcii
+PWPsyy17OP	{{I&& }}
"	 +VRV
 nnW%%>>*<= 779s6: #33`  XX224(( 54 d
## 54s   3H
Hzself: jt_allc                 J   U [         R                  R                  R                  R                  :X  a  gU [         R                  R                  R
                  R                  :X  a  US   R                  $ U [         R                  R                  R                  R                  :X  a  [        US   R                  5      $ U [         R                  R                  R                  R                  [         R                  R                  R                  R                  4;   au  US   R                  bH  [        [        US   R                  5      [        R                  " US   R                  SS  5      -  5      $ US   R                   R                  5       $ U [         R                  R                  R"                  R                  :X  a  US   R$                  $ U [         R                  R                  R&                  R                  :X  a  US   R                   R)                  5       $ g )NFr   r   )rH   opsatenis_non_overlapping_and_densedefaultsym_sizer{   r"   ra   	sym_numelnumelrP   intr   mathprodr   
sym_stride_stridessym_storage_offsetstorage_offsetrg   rh   ri   s      r   tensor_attr_supported_getterr    ss    uyy~~::BBBuyy~~&&...Aw}}uyy~~!!)))47==!!		((00%))..2F2F2N2NOO7's47++,tyyaqr9J/KKLLAw$$&&uyy~~((000Awuyy~~00888Aw--// 9r%   c                 "    [         R                  $ r   )rH   jaggedr  s      r   prim_layout_defaultr    s    <<r%   c                 ~    U [         R                  R                  R                  R                  :X  a  [        S5      eg )NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)rH   r  r  r   r  r'   r  s      r   tensor_attr_unsupported_getterr    s7    
 uyy~~""***9
 	
 +r%   c                 
   SSK Jn  [        XUSS9u  pEUR                  S5      nUR	                  5       b  gUR                  S[        R                  5      US'   US   [        R                  :X  a  gU" UR                  40 UD6$ )Nr   )is_contiguous_for_memory_formatTr   r   Fmemory_format)
r   r  r   r   r   r   rH   contiguous_formatpreserve_formatr   )rg   rh   ri   r  r   r   r   s          r   is_contiguous_generalr    s    C&TMA ..
!C {{} ",..00#J /"e&;&;;*3;;E*EEr%   z!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c                 B   [        XUSS9u  p4UR                  S5      n[        U5      nUR                  bM  US   [        R
                  :X  a6  SSKJn  UR                  S:X  d   S5       eU" UR                  5       S S9u  pU$ [        U " UR                  40 UD640 UD6$ )	NTr   r   r  r   )jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r   )r   r   r   rP   rH   r  r   r!  rQ   unbindr	   r   )	rg   rh   ri   r   r   r   new_metar!  contigs	            r   clone_defaultr%    s     'TMA ..
!Cc"H
||o&%*A*AA 8 1$MLM$(tDIFMS[[7J7D8DDr%   zinput: jt, weight: t, bias: t?c                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ NTr   r   r   r   r	   r   r   rg   rh   ri   r   r   r   s         r   linear_defaultr*  #  H    &TMA ..
!CS[[7J7O>#;NOOr%   z6self: jt, grad_output: jt, weight: t, output_mask: anyc                 D   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nSu  pn[        XSUS5        US	   (       a4  [        [        R
                  " UR                  U5      40 [        U5      D6n	US
   (       a{  UR                  R                  SUR                  S	5      5      nUR                  R                  SUR                  S
5      5      n[        R
                  " UR                  5       U5      n
US   (       al  [        [        UR                  R                  S
-
  5      5      nUS:X  a  UR                  R                  5       nO [        R                  " UR                  USS9nXU4$ )NTr   r   grad_outputr   output_mask)NNNselfr   r   rC   r   r2   F)keepdim)r   r   r|   r	   rH   matmulr   r   r   r   rU   r   ranger   cloner   )rg   rh   ri   r   r   r   r-  r   r.  dsdwdbgrad_2dinput_2dreduce_dimss                  r   linear_backward_defaultr:  .  s`   
 'TMA ..
!C../K^^H%F../K!JBB$V[-H1~LL,,f5
9G9T
 1~ %%--b&++a.A;;&&r6;;q>:\\'))+x01~ E+"5"5":":Q">?@"$$**,B;..UKBB<r%   zinput: jt_all, dtype: anyc                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r'  r(  r)  s         r   to_dtyper<  W  r+  r%   c                 j   SSK Jn  [        XUSS9u  pEUR                  S5      nUR                  S5        U " UR                  40 UD6nUR
                  R                  UR                  S9nS n	UR                  b#  UR                  R                  UR                  S9n	SS	K	J
n
  SS
KJnJn  UR                  c  UR
                  OUR                  nU	c  UOU	n[        XU45      (       a"  U" U5      nU" U5      nUR                  Ul        OX=   X>'   [!        U5      nUUS'   U	US'   [#        U40 UD6nU$ )Nr   _tensor_symint_registryTr   r   layoutdevicer   
FakeTensorFunctionalTensormb_unwrap_functional_tensorr   r   )r   r?  r   r   r   r   torB  rP   torch._subclasses.fake_tensorrD  #torch._subclasses.functional_tensorrF  rG  r   nested_int_memor   r	   )rg   rh   ri   r?  r   r   r   
new_valuesnew_offsetsnew_lengthsrD  rF  rG  ragged_source	new_thingtgtsrc
inp_kwargsr$   s                      r   to_copy_defaultrT  b  s)   6&TMA ..
!CNN8ckk0Z0J,,//):):/;KK
||llooZ->->o?8
 %(LL$8CLLcllM*2I)*:;<<))4)-8!11-D-S*$J'Jy'Jy*3
3FMr%   z-self: jt_all, src: jt_all, non_blocking: any?c                 @   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  UR                  :w  a  UR                  5       nU Vs/ s H  oR                  PM     n	nUR                  5       n
U
 Vs/ s H  oR                  PM     nnX:w  a%  [        SUR                   SUR                   35      e[        Xz5       H  u  pUR                  U5        M     UR                  R                  UR                  5        U$ s  snf s  snf )NTr   r   rR  z<copy_(): expected compatible input and src shapes, but got: rz   )	r   r   r{   r"  r   r'   r   copy_r   )rg   rh   ri   r   r   r   rR  	inp_compscinp_comp_shapes	src_compssrc_comp_shapesinp_compsrc_comps                 r   copy_defaultr^    s     'TMA ..
!C
..
C
yyCII JJL	,56Iq77I6JJL	,56Iq77I6-N99+U399+/  #&i";HNN8$ #< KKckk"J! 76s   D	Dc                    [        XUSS9u  p4UR                  S5      n[        R                  US'   U " UR                  40 UD6nUR
                  R                  UR                  S9nS nUR                  b#  UR                  R                  UR                  S9n[        U5      n	SU	;   a  XyS'   SU	;   a  XS'   UR                  UR                  :w  az  SS	K
Jn
  SS
KJnJn  SSKJn  UR                  c  UR
                  OUR                  nUc  UOUn[#        XU45      (       a"  U" U5      nU" U5      nUR$                  Ul        OX   X'   ['        U40 U	D6$ )NTr   r   r@  rA  r   r   r   rC  rE  r   r>  )r   r   rH   stridedr   r   rH  rB  rP   r   rI  rD  rJ  rF  rG  r   r?  r   rK  r	   )rg   rh   ri   r   r   r   rL  rM  rN  output_kwargsrD  rF  rG  r?  rO  rP  rQ  rR  s                     r   like_factory_defaultrb    sK    'TMA ..
!C
 !==Jxckk0Z0J,,//):):/;KK
||llooZ->->o?"3'MM!#.i M!#.i 
zzZ&&& 	=	

 	;(+(<#,,#.#6KK	i.>!?@@-i8C-m<C"%"5"5C1H1W#.
4m44r%   zself: jt_all, fill_value: anyzself: jt_all, high: anyz!self: jt_all, low: any, high: anyc                 f    [        XUSS9u  p4UR                  S5      nU " UR                  5        U$ r'  r   r   r   r)  s         r   zero__defaultre    s6    &TMA ..
!CJr%   z*self: jt_all, dim: any, half_to_float: anyc           
      d   [        XUSS9u  p4[        US   [        5      (       a  [        S5      eUR	                  S5      n[        UR                  5       US   4SUR                  5      u  US'   nnnU(       a  [        S5      eU(       a  UR                  S:  a  [        S	5      eU(       a  UR                  b  [        S
5      eUS   S   US'   U(       Ga2  [        R                  R                  R                  [        R                  R                  R                  UR                   R#                  UR                   R$                  S   S5      UR&                  /UR(                  /[+        S5      S9UR                  S9n	[        R                  R                  R-                  U	UR&                  /UR                   R$                  S   S9R"                  " S/UR                   R$                  SS  Q76 n
[/        U
40 [1        U5      D6$ [/        U " UR                   40 UD640 [1        U5      D6$ )NTr   r"   zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorr   softmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorr   zesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorr   rC   z-inf)max_lengthspadding_valuer"   r   )r   r   r   r'   r   r=   r"   rQ   rP   rH   nn
functionalrg  r  r  _jagged_to_padded_dense_forwardr   r   r   r   _max_seqlenfloat_padded_dense_to_jagged_forwardr	   r   )rg   rh   ri   r   r   r   reduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuess              r   _softmax_defaultrw    s/    'TMA *U#U++V
 	
 ..
!C 			E			5 `
 	
 COOa/s
 	
 CLL4I
 	

 #5)	Ju  % 3 3 ; ;IINN::##KK%%a("  __-#Fm ;   !< 
!
 GG!\\NKK%% H 
 ' 
 ""12&
 NBnS.ABBS[[7J7O>#;NOOr%   z7grad_output: jt, output: jt, dim: any, input_dtype: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " UR                  UR                  40 UD640 [	        U5      D6$ )NTr   r-  r$   r(  )rg   rh   ri   r   r   grad_outr$   s          r   _softmax_backwardrz  L  sh    
 'TMA ~~m,H^^H%FXv~~<<@Nx@X r%   z!self: jt, float: any, train: any?c                     [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6u  pg[        U40 [	        U5      D6[        U40 [	        U5      D64$ r'  r   r   r   r	   r   )rg   rh   ri   r   r   r   out1out2s           r   native_dropout_defaultr  [  sj     'TMA ..
!Cckk0Z0JDT1^C01T1^C01 r%   z%grad_output: jt, mask: jt, scale: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " UR                  UR                  40 UD640 [	        U5      D6$ )NTr   r-  maskr(  )rg   rh   ri   r   r   r-  r  s          r   native_dropout_backward_defaultr  k  sh    
 'TMA ../K>>&!D[  $,,=*=

% r%   z2self: jt_all, dim: any, keepdim: any?, dtype: any?c                 $    [        U SS/UQ70 UD6$ )Nr  r   _apply_reductionr  s      r   prod_dim_intr  {  s    
 D&!=d=f==r%   zself: jt_all, dtype: any?c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r'  rd  r)  s         r   prod_defaultr    9    &TMA ..
!C*z**r%   z$self: jt, split_size: any, dim: any?c                    ^ [        XUSS9u  p4UR                  S5      m[        TR                  5       US   TR                  S5      US'   [        U4S jU " TR                  40 UD6 5       5      $ )NTr   r   r"   r^   c              3   P   >#    U  H  n[        SS U0[        T5      D6v   M     g7f)valuesNr2   )r	   r   )r   rK   r   s     r   r   split_tensor.<locals>.<genexpr>  s*      0A 	5A5!450s   #&)r   r   r/   r"   rQ   r   r   r)  s        @r   split_tensorr    sy     'TMA ..
!C(	:e$coowJu  ckk0Z0  r%   z%self: jt, split_sizes: any, dim: any?c           
         [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      US'   U " UR
                  40 UD6 Vs/ s H  n[        SSU0[        U5      D6PM     sn$ s  snf )NTr   r   r"   split_with_sizesr  r2   r   r   r/   r"   rQ   r   r	   r   )rg   rh   ri   r   r   r   rK   s          r   split_with_sizes_defaultr    s     'TMA ..
!C(	:e$coo7IJu ckk0Z00A 	5A5!450  s     Bz+self: jt, dim: any, start: any, length: anyc                     [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      nU " UR
                  UUS   US   S9n[        U40 [        U5      D6$ )	NTr   r   r"   narrowstartlengthr"   r  r  r  )rg   rh   ri   r   r   r   r"   r  s           r   r  r    s     'TMA ..
!C
3779j&7(
SC!(#	F 6."566r%   z self: jt, chunks: any, dim: any?c           
         [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  SSS9u  US'   nU(       Ga  US   nUR
                  R                  5       nUR                  U5      n	U	 V
s/ s H  n
[        R                  " U
SS	9PM     nn
U V
s/ s H  n
[        R                  " U
S
SS9PM     nn
U Vs/ s H  nXR                  S.PM     nnU	 V
s/ s H   oR                  5       R                  5       PM"     nn
UR                  R                  U5      n[!        S[#        U5      5       Vs/ s H  n[%        SSUU   0UU   D6PM     sn$ U " UR                  40 UD6 V
s/ s H  n
[%        SSU
0['        U5      D6PM     sn
$ s  sn
f s  sn
f s  snf s  sn
f s  snf s  sn
f )NTr   r   r"   chunkr+   chunksr   rj  r   r   )value)r   rQ   r  r2   )r   r   r/   r"   rQ   r   diffr  rH   cumsumFpadr   itemr   r^   r2  ra   r	   r   )rg   rh   ri   r   r   r   r.   r  r   chunked_lengthsrK   chunked_offsetsper_offsetsnested_kwargssplit_sizeschunk_valuesrn   s                    r   chunk_defaultr    s   &TMA ..
!C,<	:e$coowPT-)Ju) H% ,,##%!--/;JK?a5<<q1?K>MNo155F!4oN  /
. $OOD. 	 
 0??!uuw||~?{{((5 1c,/0
0 FQFM!4DF0
 	
 #++44
4 99^C%894
 	
% LN
 @



s$   	F%. F*F/2'F4F9 F>zself: jt_all, dim: any?c           
        ^^ [        XUSS9u  p4US   nUS:w  a  [        S5      eUR                  S5      nUR                  5       mUR	                  5       nUR                  5       nUR                  mSS[        [           S[        [        [              4UU4S	 jjjn	Uc?  UR                  5       R                  5       n
U	" U
5        [        R                  " TU
TS
-
  S9$ TS::  a  [        S5      eUR                  5       n
UR                  5       nU	" X5        [        UR                  S   5       Vs/ s H!  n[        R                   " TTS
-
  X   X   S9PM#     sn$ s  snf )NTr   r"   r   z2unbind(): only supported for NestedTensor on dim=0r   rP   r   c                   > SnTR                   TS-
     n[        [        U 5      5       Hd  n[        R                  " X   5        [        R
                  " X   U:*  5        X U   -  nUc  MB  [        R
                  " X   X   -   U:*  S 5        Mf     [        R
                  " X#:*  5        UbO  [        [        U5      5       H6  n[        R                  " X   5        [        R
                  " X   U:*  5        M8     g g )Nr   r   c                      g)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionr2   r2   r%   r   rL   2unbind_int.<locals>._torch_check.<locals>.<lambda>  s    kr%   )r   r2  ra   rH   _check_is_size_check)rP   r   lengths_sumragged_dim_sizern   r3   r  s        r   _torch_check unbind_int.<locals>._torch_check  s      ,,zA~6s8}%A  -LL78A;&K#K(+-@k & 	[343x=)$$X[1X[O;< *  r%   r   rj  zAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)r  r   )r   r'   r   r  r   r   rQ   r   r  r   r  tolistrH   r^   r2  r   r  )rg   rh   ri   r   r   r"   r   r   r   r  lengths_scalarsoffsets_scalarsrn   r3   r  s                @@r   
unbind_intr    s_    'TMA U
C
axOPP
..
!CZZ\FkkmGkkmGJ=tCy =HT#Y4G = =: !,,.//1_%{{6?aIIQO
 	
 nn&Onn&O2 w}}Q'( )A 	a!$"%		
 )  s   4(Ezself: jt, dim: anyc                     [        XUSS9u  p4UR                  S5      nUR                  n[        [	        UR
                  5      US   UR                  S5      US'   [        U " U40 UD640 [        U5      D6$ )NTr   r   r"   r   )	r   r   r   r/   ra   r{   rQ   r	   r   )rg   rh   ri   r   r   r   r  s          r   squeeze_dimr  <  sx    &TMA ..
!C[[F(CII
5)3??IJu V2z2JnS6IJJr%   zself: jt_all, dim: anyc                 @   [        XUSS9u  p4UR                  S5      nUR                  nUS   n[        [	        UR
                  5      S-   XuR                  SSS9US'   [        U5      nUS   UR                  S-
  ::  a  US==   S-  ss'   [        U " U40 UD640 UD6$ )	NTr   r   r"   r   	unsqueeze)r*   rQ   )	r   r   r   r/   ra   r{   rQ   r   r	   )	rg   rh   ri   r   r   r   r  r"   ra  s	            r   unsqueeze_defaultr  K  s    &TMA ..
!C[[F U
C(CIIC+PTJu
 #3'M%COOa//m$)$V2z2DmDDr%   ztensors: any, dim: anyc                    [        XUSS9u  p4UR                  S5      nU Vs/ s H  ofR                  (       d  M  UPM     nn[        U5      S:  d   eUS   nU Vs/ s H&  ofR                  (       a  UOUR	                  U5      PM(     nnUS   n	[        [        UR                  5      XR                  S5      US'   [        U " U Vs/ s H  ofR                  PM     sn40 UD640 [        US   5      D6$ s  snf s  snf s  snf )NTr   tensorsr   r"   cat)r   r   	is_nestedra   	expand_asr/   r   rQ   r	   r   r   )
rg   rh   ri   r   r   r  rU   nestedfirstr"   s
             r   cat_defaultr  b  s    &TMA nnY'G !0AKKaF0v;??1IEAHIAKKqQ[[%77GI U
C(EKK#00%Ju )Aii)8Z8<J7ST:<V  1 J *s   C7C7-C<	Dzself: any, other: anyc                   ^  [        T XSS9u  p4UR                  S5      nUR                  S5      nU 4S jnU 4S jnUR                  (       a  UR                  (       d  UR                  5       S:  aK  UR                  5       UR                  5       :X  a)  UR                  UR                  5       S-
  :  a  U" XV5      $ UR                  5       S	:X  aj  UR                  5       UR                  5       :  aH  UR                  UR                  5       S-
  :  a'  [        T " UR                  U40 UD640 [        U5      D6$ GOUR                  (       d  UR                  (       a  UR                  5       S:  a:  UR                  5       UR                  5       :X  a  UR                  S	:  a  U" XV5      $ UR                  5       S	:X  aX  UR                  5       UR                  5       :  a6  UR                  S	:  a&  [        T " XVR                  40 UD640 [        U5      D6$ GO2UR                  (       Ga   UR                  (       Ga  UR                  5       S:  a^  UR                  5       S:  aJ  [        XVR                  5      (       a0  [        T " UR                  UR                  5      40 [        U5      D6$ UR                  5       S:X  a  UR                  5       S:X  at  UR                  S	:X  ad  UR                  S:X  aT  UR                  UR                  5      UR                  UR                  5      :X  a  [        R                  " U" XV5      5      $ [        S
UR                   SUR                   35      e)NTr   r   otherc                    > [        U R                  5       UR                  5       5       VVs/ s H  u  p#T" X#5      PM     snn$ s  snnf r   )r   r"  )ru   rw   a_compb_comprg   s       r   _unbind_impl$matmul_default.<locals>._unbind_impl  s@    9<QXXZ9T
9T%5fD 9T
 	
 
s   Ac           	        > U R                   (       a  U nOUnSSKJn  UR                  nUR                  nUnUR
                  R                  UR                  S-
     nUc  Un/ UR                  S UR                   QUPUR                  UR                  S-   S  Q7nUR                  SUS9n	U R                   (       a	  T" X5      n
OT" X	5      n
U" U
UR                  UR                  UUUS9$ )Nr   r           output_sizer   )
r  r   r   r   r   r   r   rQ   to_padded_tensorr   )ru   rw   r   r   r   r   r   r   padded_shape	padded_ntpadded_trg   s              r   _padded_impl$matmul_default.<locals>._padded_impl  s    ;;BB5))
))
!**""2>>A#56"L
XX&'

 XXbnnq(*+

 '''F	;;I)HA)H!KK~~!!
 	
r%      r   r   z1matmul(): not supported between inputs of shapes rz   )r   r   r  r"   rQ   r	   r   r   r   r{   r   rH   stackr'   r   )	rg   rh   ri   r   r   r   r  r  r  s	   `        r   matmul_defaultr  {  s   &4TMA ..
!CNN7#E

!
J }}U__ GGIN	UYY[(#'')a-/  ++ IIK1	EIIK'#'')a-/S[[%6:6:H:M  ]]u99;!		swwy 8U=N=NRS=S++
 WWY!^		cggi 7E<M<MQR<RS--6:6:H:O 
 
5??? 779q=UYY[1_1CC1U1US[[%-- @XNSVDWXX GGIN		q 1$!!Q&)UZZ8I8I-JJ ;;|C788

;CII;eEKK=Y r%   zself: jt_all, mat2: anyc                 L   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  5       S:w  a  [        S5      eUR                  5       S:w  a  [        S5      e[	        [
        R                  R                  R                  R                  XV5      $ )NTr   r   mat2r  zbmm(): input must be 3Dzbmm(): mat2 must be 3D)
r   r   r"   rb   r  rH   r  r  r1  r  rg   rh   ri   r   r   r   r  s          r   bmm_defaultr    s    &TMA ..
!CNN6"E
wwyA~233yy{a122%))..//77DDr%   z'self: jt_all, size: any, implicit: any?c                    [        XUSS9u  p4UR                  S5      nUS   nSU;  d  UR                  S5      (       a   e[        XV5      (       d  [        SUR                   SU 35      e[        SUR                  5       5       Vs/ s H  owUR                  :X  a  S	OXg   PM     nn[        U " UR                  U5      40 [        U5      D6$ s  snf )
NTr   r   r   implicitzexpand(): cannot expand shape z -> r   rC   )r   r   r   r'   r{   r2  r"   rQ   r	   r   r   )	rg   rh   ri   r   r   r   r   r   
expand_args	            r   expand_defaultr    s     'TMA ..
!CfDj(*..2L2LMMc((;CII;d4&QRRCHCGGICVWCVaS__,"$'9CVJWS[[*5M9LMM Xs   	Czself: t, other: jtc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " XVR                  5      40 [	        U5      D6$ )NTr   r   r  r(  r  s          r   expand_as_defaultr    sQ    &TMA ..
!CNN7#ES--0JN54IJJr%   zself: jt_all, size: anyc           
      .   [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      UR                  5       ::  a?  UR	                  / S [        UR                  5       [        U5      -
  5       5       QUQ5      $ [        S5      e)NTr   r   r   c              3   &   #    U  H  nS v   M	     g7f)r   Nr2   )r   r   s     r   r   broadcast_to.<locals>.<genexpr>#  s     E(D1Q(Ds   zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)r   r   ra   r"   r   r2  rb   )rg   rh   ri   r   r   r   r   s          r   broadcast_tor    s    &TMA ..
!C>>&!D
4yCGGIzzMEcggi#d).C(DEMMNN
	4 r%   ztensors: anyc           
      b   [        XUSS9u  p4UR                  S5      n[        U5      S:X  a  [        S5      e[        U5      S:X  a  US   $ / n[        R
                  " S U 5       6 n[        S U 5       5      nU H  n	U	R                  (       a"  UR                  U	R                  U5      5        M6  U	R                  5       [        U5      :  aI  UR                  [        U	R                  UR                  R                  5      40 [        U5      D65        M  [        S	5      e   [        U5      $ )
NTr   r  r   z7broadcast_tensors(): expected at least one tensor inputr   c              3   8   #    U  H  oR                   v   M     g 7fr   )r   r   rU   s     r   r   $broadcast_tensors.<locals>.<genexpr>8  s     .H1wws   c              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rS   r  s     r   r   r  :  s     A'QZ<%@qq'r   zubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)r   r   ra   rb   rH   broadcast_shapesr   r  r   r  r"   r	   r   r   r   r   )
rg   rh   ri   r   r   r  outsbroadcast_shaper   rU   s
             r   broadcast_tensorsr  +  s	   &TMA nnY'G
7|qRSS
7|qqzD,,.H.HIO
A'A
AC;;KK78UUWs?++KKQ^^CKK,=,=>V.QTBUV ;   ;r%   z(condition: jt_all, self: any, other: anyc                 .   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S5      n[        R                  " XVU5      u  pVn[	        U " UR
                  UR
                  UR
                  40 UD640 [        U5      D6$ )NTr   	conditionr   r  )r   r   rH   r  r	   r   r   )rg   rh   ri   r   r   r  r   r  s           r   
where_selfr  K  s     'TMA {+I
..
!CNN7#E "33IEJIEYU]]IjI

# r%   zself: jt, device: any?c                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r'  r(  r)  s         r   _pin_memory_defaultr  `  r+  r%   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r'  rd  r)  s         r   is_pinned_defaultr  k  r  r%   zself: jt_all, other: jt_allc                 @    US   R                   US   R                   :H  $ r   )r{   r  s      r   is_same_size_defaultr  v  s     7==DGMM))r%   c                   ^^ [        XUSS9u  pVUR                  S5      mUS   S L =(       d1    [        US   [        [        45      =(       a    [        US   5      S:H  nU(       a  U " TR                  40 UD6nUR                  SS5      (       aS  [        U[        [        45      (       a  [        U5      " U4S jU 5       5      nU$ UR                  TR                  5      nU$ US   n	[        US   [        [        45      n
U
(       d  U	/n	[        TR                  5       U	U TR                  5      u  nnnnU
(       d  US   nXS'   U(       a  TR                  b  [        U S	35      eSS
KJn  U(       a  U(       a6  U " TR                  40 UD6nUR                  SS5      (       a
  U" S U5      nU$ U(       a  [        U S35      eUR                  S5        U
(       a  TR                  /OTR                  nU " TR#                  U5      4SU0UD6$ U(       a  [        U S35      eU " TR                  40 UD6n[%        T5      mUR                  SS5      (       dT  [        US   [        [        45      (       a  US   OUS   /nU H%  nUTR                  S-
  :  d  M  TS==   S-  ss'   M'     U" U4S jU5      $ )NTr   r   r"   r   r0  Fc              3   X   >#    U  H  oR                  TR                  5      v   M!     g 7fr   )r  rQ   )r   or   s     r   r   #_apply_reduction.<locals>.<genexpr>  s     JcCOO < <cs   '*zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapc                 $    U R                  S5      $ )Nr   )r  )r  s    r   rL   "_apply_reduction.<locals>.<lambda>  s    Qr%   zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsr   rQ   c                    > [        U 40 TD6$ r   r   )r  
out_kwargss    r   rL   r    s    ,q"?J"?r%   )r   r   r   r   r   ra   r   r   r   r  rQ   r=   r"   rP   r'   torch.utils._pytreer  r  r   )rg   	func_nameidentity_elementrh   ri   r   r   full_reductionoutdim_to_convert
is_dimlistconverted_dimrr  rs  reduce_on_non_batchr  dim_to_passdimlistr   r   r  s                      @@r   r  r  }  s   &TMA ..
!C  &$. :e$udm4TZ=N9OST9T  3;;-*->>)U++#t}--3iJcJJ 
 mmCOO4
  &NJu-t}=J() 		+		 %a(%uCLL4k ; ;
 	

 - s{{1j1C~~i//7=J #" k ": :  NN5!/93??+sK$$%56<GKU  + K K  3;;-*-#C(
~~i// j/%?? 5! '( 
 s**}-2-  ?EEr%   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r'  rd  r)  s         r   sum_defaultr    r  r%   z3self: jt_all, dim: any?, keepdim: any?, dtype: any?c                 $    [        U SS/UQ70 UD6$ )Nr   r   r  r  s      r   sum_dim_IntListr    s    
 D%<T<V<<r%   z"self: jt_all, dim0: any, dim1: anyc           
      (   [        XUSS9u  p4SSKJn  UR                  S5      nU" UR	                  5       US   US   45      u  pxXvR
                  :X  d  XR
                  :X  a  US:X  d  US:X  a  [        S5      eXvR
                  :X  a  Un	OUn	[        U5      n
XS	'   [        UR                  5       R                  [        [        UR                  5      XvR
                  5      [        [        UR                  5      XR
                  5      5      40 U
D6$ [        UR	                  5       US   UR
                  S
5      US'   [        UR	                  5       US   UR
                  S
5      US'   [        U " UR                  40 UD640 [        U5      D6$ )NTr   r   r   r   dim0dim1z?Transpose is not supported on the batch dimension for jagged NTrQ   	transpose)r   r   r   r   r"   rQ   rb   r   r	   r  r  r   ra   r{   r/   r   )rg   rh   ri   r   r   r   r   r  r  to_dimrS  s              r   transpose_intr    su    'TMA 6
..
!C"3779z&/A:fCU.VWJD
 $//"919	Q  ??"FF#C(
$*=!JJL""#C		ND//J#C		ND//J

 
 	
 *	:f%sJv *	:f%sJv S[[7J7O>#;NOOr%   zself: jt_all, dims: anyc                    [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      n[        UR                  5      nU[        U5      :w  a  [        SU S3S[        U5       S3-   5      eS	S
KJn	  U	" X5      n
[        U
5      [        [        U
5      5      :w  a  [        S5      eUR                  b  [        S5      eU
S	   S	:w  a  [        S5      eU
R                  UR                  5      US'   U
SS   Vs/ s H  n[        XUR                  5      PM     nnXS'   [        U " UR                  40 UD640 UD6$ s  snf )NTr   r   r7   z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().r   r   z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrQ   r   )r   r   r   ra   r{   rb   r   r   setrP   indexrQ   r   r	   r   )rg   rh   ri   r   r   r   r7   rS  inp_dimr   canonicalized_dimsr"   
inner_dimss                r   permute_defaultr  '  sv   &TMA ..
!C>>&!D$J#))nG #d)CG9BOQRUVZR[Q\\^_`
 	

 6*79
#c*<&=">>EFF
||P
 	
 !!K
 	
 !3 8 8 IJ} &ab))C 	G#//:)   $vS[[7J7F:FFs   	 Ec                   ^^	 [        XUSS9u  p4UR                  S5      mUR                  S5      m	TR                  S:w  a>  [        TR                  5      [        T	5      :w  a  [        STR                   ST	 S35      e[        T	5      S	:  d  [        TT	5      (       d  [        S
TR                   ST	 35      eUU	4S jn[        [        T	5      S-
  5       Vs/ s H
  oe" U5      PM     nn[        R                  " TR                  5       5         [        U " TR                  U5      40 [        T5      D6sS S S 5        $ s  snf ! , (       d  f       g = f)NTr   r   r   r   zVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r  r  zview(): cannot view shape z as c                 p   > U TR                   S-
  :X  a  TR                  R                  U 5      $ TU S-      $ rO   )rQ   r   r   )	inner_idxr   r   s    r   get_inner_size$view_default.<locals>.get_inner_sizet  s8    !++;;##I..	A&&r%   )r   r   rQ   r   r{   r'   ra   r   r2  rH   inference_modeis_inferencer	   r   r   )
rg   rh   ri   r   r   r"  rn   
inner_sizer   r   s
           @@r   view_defaultr'  P  s2   
 'TMA ..
!C>>&!D
!cii 0E$K ? YYKtfB@
 	
 4y1}.sD997		{$tfMNN ' .33t9q=-AB-A.#-AJB 
		c..0	1Dj9Q^C=PQ 
2	1	 C 
2	1s   E&E
EzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anyc                 &   [        XUSS9u  p4UR                  S5      nUR                  5       S::  a  [        S5      eUS   nUR                  UR
                     nUR                  5       [        U5      -
  nUS:X  a  [        S5      eXv;   a  UR                  b  [        S	5      eXv;   Ga:  [        R                  R                  R                  UR                  R                  UR
                  S
9UR                  /UR                  /S9n	[        R                  R                  R                  [        R                   " UR                  R                  S   S4UR"                  UR$                  S9UR                  /UR                  /S9R'                  U	R                  5      n
UR                  R)                  5       R+                  S5      R+                  S5      U	R                  S   -  n[        R,                  " U	SSS9U-  nX-
  U
-  n[        R,                  " [        R.                  " U5      SSS9U-  n[        R0                  " XS   -   5      nX-  n[        R                  R                  R3                  UUR                  /UR                  R                  S   S9R5                  SUR                  UR
                  S-   S  5      n[7        U40 [9        U5      D6UU4$ U " UR                  40 UD6u  np[7        U40 [9        U5      D6X4$ )NTr   r   r   zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsr   r   zVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr   )rh  r   )rB  dtype)r   r   )r"   r0  r   rk  rC   )r   r   r"   r'   r   rQ   ra   rP   rH   r  r  rn  r   r   r   ro  onesrB  r*  r   r  r  r   squaresqrtrq  	unflattenr	   r   )rg   rh   ri   r   r   r   r   ragged_sizenum_dims_not_normalizedpadded_inputpadded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr$   s                      r   native_layer_norm_defaultr:    s   
 'TMA ..
!C
wwyA~]
 	
 ""45))COO,K!ggi#.>*?? 	 1$d
 	
 &3<<+Cy
 	

 	'yy~~EEKK//    \\N) F 
 iinnDDJJ))!,a0399U\\N) E 
 &
	 	 LL))!,66q9L<N<Nq<QQ 	
 II
  	 
 II./
  	 jju$556-3#(99>>#Q#Q\\NKK%% $R $
 )		#//A-/0
 	! 1I^C5HI
 	
 S[[7J7FD7>##67CCr%   zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      nU " UR                  UR                  40 UD6u  pxn	Uc  S X4$ [        U40 [	        U5      D6X4$ )NTr   ry  r   r|  )
rg   rh   ri   r   r   ry  r   d_inputd_gammad_betas
             r   "native_layer_norm_backward_defaultr?    s    
 'TMA ~~j)H
..
!C#H$4$4ckkPZPGfg&&8N3$78'JJr%   z"self: jt_all, dim: any, index: anyc                    [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  SSS9u  US'   nU(       a  UR                  5       US      $ UR                  b  [        S5      e[        U5      nUS   UR                  S	-
  :  a  US
==   S	-  ss'   [        U " UR                  40 UD640 UD6$ )NTr   r   r"   selectr  r  zSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesr   rQ   )r   r   r/   r"   rQ   r"  rP   rb   r   r	   r   )rg   rh   ri   r   r   r   r.   r  s           r   
select_intrB    s    &TMA ..
!C,<	:e$cooxQU-)Ju) zz|Jw/00
||a
 	

  $J%3??Q..=!Q&!S[[7J7F:FFr%   z7self: jt, dim: any?, start: any?, end: any?, step: any?c                     [        XUSS9u  p4UR                  S5      n[        UR                  5       US   UR                  S5      US'   [        U " UR                  40 UD640 [        U5      D6$ )NTr   r   r"   slice)r   r   r/   r"   rQ   r	   r   r   r)  s         r   slice_tensorrE    ss    
 'TMA ..
!C(	:e$coowJu S[[7J7O>#;NOOr%   z8input: jt_all, indices: any, values: t, accumulate: any?c           	         [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U5      UR                  5       ::  d   e[        U5      UR                  S-   :  Ga:  UR                  5       (       d  [        S5      eSSKJn  UR                  nUR                  n	U	n
UR                  R                  UR                  S-
     nU
c  Un
/ UR                  S UR                   QU
PUR                  UR                  S-   S  Q7nUR                  SUS	9nU" U " X40 UD6UR                  UR                  UUU	S
9nU [        R                   R"                  R$                  R&                  :X  a+  UR                  R)                  UR+                  5       5        U$ U$ UR-                  5       c  UR/                  5       R1                  5       nOUR-                  5       n[        R2                  " [        R4                  " XeR                     U:  5      S5        UR/                  5       US      XeR                     -   nUSUR                   U/-   XeR                  S-   S  -   nU [        R                   R"                  R$                  R&                  :X  a  U " UR                  U40 UD6Ul        U$ [7        U " UR                  U40 UD640 [9        U5      D6$ )NTr   r   indicesr   z[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsr   r  r  r   z7Some indices in the ragged dimension are out of bounds!r   )r   r   ra   r"   rQ   is_contiguousr'   r   r   r   r   r   r   r  r   rH   r  r  
index_put_r  rV  r  r   r   r  _assert_asyncr   r	   r   )rg   rh   ri   r   r   r   rG  r   r   r   r   r   r  
padded_inpnew_njtr   ragged_indicesfunc_indicess                     r   rI  rI  '  s    'TMA #w/C nnY'Gw<3779$$$
7|coo))  ""m  	6**
**
!++##COOa$78"L
YY()

 YYs*,-

 ))#<)H
$3
3LL!!
 599>>,,444KKgnn./J
 {{}++-$$&++-			'//*W45A [[]71:.1IIN 	COO$
	 //A%'
(		)  uyy~~((0003;;C
C
S[[,5*5

 r%   z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anyc                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r'  r(  r)  s         r   convolution_defaultrP    sJ     'TMA ..
!CS[[7J7O>#;NOOr%   c                 r   [        XUSS9u  p4US   n[        UR                  5       US   SUR                  5      u  p6pxU(       a  U(       d  U(       a   eUS   n	SUS'   [	        [
        R                  R                  R                  R                  SS40 UD6n
UR                  b  UR                  OUR                  R                  5       n[        U
R                  5       S-
  5       H  nUR                  S	5      nM     X-  nU	(       d  UR                  UR                  5      nU$ S
n[	        U SU40 UD6$ )NTr   r   r"   r4  r0  r   r   rC   gzG?)r   r=   r"   rQ   r  rH   r  r  r   dim_IntListrP   r   r  r2  r  r   )rg   rh   ri   r   r   r   rr  rs  r
  r0  intermediate_sumr   r  intermediate_values                 r   mean_dimrU    s1    'TMA W
CBS	5	C?Q) &&&Y' $
9+IINN**FA
9C

 #&,,":#,,@Q@Q@S'++-12A''+G 3(++coo.C

 D&*<K
KKr%   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r'  rd  r)  s         r   mean_defaultrW    r  r%   z&self: jt_all, dim: any?, keepdim: any?c                 $    [        U SS/UQ70 UD6$ )Nr6   Fr  r  s      r   any_dimsrY    s    D%@@@@r%   z%self: jt_all, dim: any, keepdim: any?c                     [        XUSS9u  p4US   /US'   [        [        R                  R                  R
                  R                  40 UD6$ NTr   r"   )r   rY  rH   r  r  r6   r7   rg   rh   ri   r   r   s        r   any_dimr]    M    &TMA
 $E*+JuEIINN&&++:z::r%   c                 $    [        U SS/UQ70 UD6$ )Nr   Tr  r  s      r   all_dimsr`    s    D%????r%   c                     [        XUSS9u  p4US   /US'   [        [        R                  R                  R
                  R                  40 UD6$ r[  )r   r`  rH   r  r  r   r7   r\  s        r   all_dimrb    r^  r%   c                 d    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6$ r'  rd  r)  s         r   all_any_max_min_defaultrd    s;     'TMA ..
!C*z**r%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   minr   rH   finfor*  maxr  rg   rh   ri   r   r   	dtype_maxs         r   min_dimrl    O    &TMA Jw/556::ID%DTDVDDr%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   ri  r   rH   rh  r*  rf  r  rg   rh   ri   r   r   	dtype_mins         r   max_dimrr    rm  r%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   aminrg  rj  s         r   amin_defaultru    Q     'TMA Jw/556::ID&)EdEfEEr%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   amaxro  rp  s         r   amax_defaultry    rv  r%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   argminrg  rj  s         r   argmin_defaultr|  %  Q     'TMA Jw/556::ID(IGGGGr%   c                     [        XUSS9u  p4[        R                  " US   R                  5      R                  n[        U SU/UQ70 UD6$ )NTr   r   argmaxro  rp  s         r   argmax_defaultr  1  r}  r%   zAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc                   ^
 SSK Jm
  [        XUSS9u  p4UR                  S5      nUR                  US'   UR                  S5      nUR                  US'   [        U
4S j[        US   5       5       5      n[        [        US   5      US	   US
5      US	'   UR                  S5      nUR                  R                  UR                  S-
  5      X'   USS  nXS'   [        U5      n	XyS'   [        U " S0 UD640 U	D6$ )Nr   r   Tr   gradrG  c              3   J   >#    U  H  u  pT" U5      (       d  M  Uv   M     g 7fr   r2   )r   rn   r   r   s      r   r   =value_selecting_reduction_backward_default.<locals>.<genexpr>M  s     W$BDAmTUFVaa$Bs   #	#sizesr"   "value_selecting_reduction_backwardr   rQ   r2   )r   r   r   r   r   r   rd   r/   ra   r   rQ   r   r	   )rg   rh   ri   r   r   r  rG  r3   r  ra  r   s             @r   *value_selecting_reduction_backward_defaultr  =  s	   
 D&TMA >>&!DJvnnY'G#OOJyWIj.A$BWWJ(Jw 5,	Ju NN7#E,,W-@-@1-DEE!"IEw"7+M#-- *z*<m<<r%   c                 &   [        XUSS9u  p4UR                  S5      nU H{  n[        U[        5      (       d  [	        S5      eUR                  5       US   R                  5       :w  a  [	        S5      e[        XeS   R                  5      (       a  Mr  [	        S5      e   [        US   R                  5       S-   US	   US   R                  S
5      US	'   [        U " U Vs/ s H  ofR                  PM     sn40 UD640 [        US   5      D6$ s  snf )NTr   r  z+stack(): expected all nested tensors inputsr   z9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurer   r"   r  )r   r   r   r	   r'   r"   r   r   r/   rQ   r   r   )rg   rh   ri   r   r   r  rU   s          r   stack_defaultr  a  s   &TMA
 nnY'G!\**LMM557gajnn&&K  "!QZ%5%566X   )
1j/1G1GJu )Aii)8Z8<J7ST:<V )s    DzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?c                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        U " XeR                  40 UD640 [	        U5      D6$ )NTr   rG  r   r(  )rg   rh   ri   r   r   rG  r   s          r   embedding_defaultr    s`    
 'TMA
 nnY'G^^H%FV__3
37Eg7N r%   zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      nU " UR                  UR                  40 UD6$ )NTr   rG  r-  rd  )rg   rh   ri   r   r   rG  r-  s          r    embedding_dense_backward_defaultr    sQ    
 'TMA nnY'G../K##W__C
CCr%   c                 r    [        XUSS9u  p4UR                  S5      nUR                  R                  5       $ r'  )r   r   r   detachr)  s         r   values_defaultr    s<     'TMA ..
!C ;;r%   c                 b    [        XUSS9u  p4UR                  S5      nU " UR                  5      $ r'  rd  r)  s         r   all_defaultr    s4    &TMA ..
!Cr%   z-self: jt_all, padding: any, output_size: any?c                    [        XUSS9u  p4UR                  S5      nUR                  b  [        S5      eUS   nUb  XeR                     nO4UR
                  b  UR                  OUR                  R                  S5      nUR                  5       nUR                  S:  a  UR                  UR                  S-
  S5      nUR                  n	UR                  5       S:  a  UR                  SS	9nO%UR                  5       S:X  a  UR                  S
5      nUR                  [         R"                  L n
U
(       a0  UR$                  (       a  UR'                  [         R(                  5      n[         R*                  R,                  R/                  UUR0                  /U/US   5      nU
(       a0  UR$                  (       a  UR'                  [         R"                  5      n[3        U	5      S:  a  UR5                  S
U	SS  5      nO [3        U	5      S:X  a  UR7                  S
5      nUR                  S:  a  UR                  UR                  S5      nU$ )NTr   r   z?to_padded_tensor(): not supported for nested tensors with holesr  r   r   r   r)  rC   padding)r   r   rP   r'   rQ   _max_seqlen_tensorro  r   r   r  r  r   r"   r   r  r*  rH   boolis_cudarH  halfr  r  rn  r   ra   r.  r   )rg   rh   ri   r   r   r   r  max_seq_lenr  values_shapeis_bool
padded_outs               r   to_padded_tensor_defaultr    s   
 'TMA ..
!C
||M
 	

 ]+K!//2 %%1 OO!!!$ 	 ZZ\F
!!#//A"5q9<<Lzz|a!,		!!"% llejj(G6>>5::&??		9	J :%%]]5::.
 <1))"l12.>?
	\	a	''+

))#//1=
r%   zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?c                    [        XUSS9u  p4US   US   peUR                  SS5      nUS:  a  UR                  US5      nUR                  nUR	                  5       S:  a  UR                  SS	9nO%UR	                  5       S:  a  UR                  S
5      nUR                  [        R                  L n	U	(       a0  UR                  (       a  UR                  [        R                  5      n[        R                  R                  R                  XV/US   5      n
U	(       a0  U
R                  (       a  U
R                  [        R                  5      n
[!        U5      S:  a  U
R#                  S
USS  5      n
O [!        U5      S:  a  U
R%                  S
5      n
US:  a  U
R                  US-
  S5      n
US   nUS   n0 nUb  XS'   Ub  XS'   ['        U
UUUS9$ )NTr   paddedr   r3   r   r  r   r)  rC   r   r   r   r   )rQ   r   )r   r   r  r   r"   r   r  r*  rH   r  r  rH  r  r  r  rq  ra   r.  r   r	   )rg   rh   ri   r   r   r  r   r3   padded_ragged_dim1_shaper  r  r   r   metadata_caches                 r   "_nested_from_padded_tensor_defaultr    s   
 'TMA !*Jy,AGa0J A~!!*a0%||zz|a!,		!!"% llejj(G6>>5::&YY^^;;	:g.F 6>>5::& #$q(!!"&>qr&BC	%	&	*#A~!!*q.!4L)JL)JN'1|$'1|$&	 r%   zcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c                     [        XUSS9u  p4US   US   US   pvnUS   nUS   n	US   n
0 nU	b  XS'   U
b  XS'   [        UUUUUS	9$ )
NTr   r   r   r   r3   r   r   )r   rQ   r   )r   r	   )rg   rh   ri   r   r   r  r   r   r3   r   r   r  s               r    _nested_view_from_jagged_defaultr  6	  s    
 'TMA
 	799 F
 L)JL)JL)JN'1|$'1|$& r%   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r'  )r   r   r   r)  s         r   _nested_get_offsetsr  V	  /    &TMA ..
!C<<r%   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r'  )r   r   rP   r)  s         r   _nested_get_lengthsr  `	  r  r%   c                 V    [        XUSS9u  p4UR                  S5      nUR                  $ r'  )r   r   rQ   r)  s         r   _nested_get_ragged_idxr  j	  s/    &TMA ..
!C??r%   c                 v    [        XUSS9u  p4UR                  S5      nUR                  R                  SS 5      $ )NTr   r   r   r   r   r   r   r)  s         r   _nested_get_min_seqlenr  t	  >    &TMA ..
!C""<66r%   c                 v    [        XUSS9u  p4UR                  S5      nUR                  R                  SS 5      $ )NTr   r   r   r  r)  s         r   _nested_get_max_seqlenr  ~	  r  r%   zself: jt, mask: anyc                    [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S:  a  [        S5      eUR                  UR                  :w  a%  [        SUR                   SUR                   35      eUR
                  R                  UR                  5       5      n[        R                  " UR                  5       R                  S	S
9S5      n[        U5      nXR                     US'   [        SSU0UD6$ )NTr   r   r  r   z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape r   rj  r  r   r  r2   )r   r   r   r'   r   r   masked_selectr  r  r  r  r   r   r	   )	rg   rh   ri   r   r   r   r  
res_valuesmask_cumsums	            r   masked_select_defaultr  	  s    &TMA ..
!C>>&!D
xx!|PQQ	djj	 tzzl*PQTQZQZP[\
 	
 **4;;=9J%%,,,3V<K#D!,,/DO 
 r%   z2grad_output: t, self: jt_all, dim: any, index: anyc                     [        XUSS9u  p4UR                  S5      nUR                  S5      n[        R                  " XVR                  S9nUR                  US   US   5      R                  U5        U$ )NTr   r   r-  )r*  r"   r  )r   r   rH   
zeros_liker*  rA  rV  )rg   rh   ri   r   r   r   r-  
grad_inputs           r   _nested_select_backward_defaultr  	  sv    
 'TMA ..
!C../K!!#->->?Jj'G)<=CCKPr%   zself: jt_all, s: anyc                     US   nUS   nU " UR                   U5        U " UR                  U5        UR                  b  U " UR                  U5        g g r   )r   r   rP   )rg   rh   ri   r   streams        r   record_stream_defaultr  	  sM    
q'C!WFfv
||S\\6"  r%   zRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?c                     [        XUSS9u  p4UR                  S5      n[        US   5      S:X  a  U " UR                  40 UD6$ [	        S5      e)NTr   r   r   r   z2new_empty() not supported for NJT with shape != ())r   r   ra   r   r'   r)  s         r   new_empty_defaultr  	  sY     'TMA ..
!C
:f!#CKK.:..
K
LLr%   zself: jt_all, ...c                 j    [        S U 5       5      n[        U " S U 5       0 UD640 [        U5      D6$ )Nc              3   T   #    U  H  n[        U[        5      (       d  M  Uv   M      g 7fr   rS   r   s     r   r   &activation_backward.<locals>.<genexpr>	  s     Ldsjl.Kssdr   c              3   h   #    U  H(  n[        U[        5      (       a  UR                  OUv   M*     g 7fr   )r   r	   r   r   s     r   r   r  	  s%     TtZ\::ckkCts   02r   )rg   rh   ri   r-  s       r   activation_backwardr  	  sH     LdLLKTtT	
	

 
% r%   zself: jt_all, value: anyc                     [        XUSS9u  p4UR                  S5      n[        U " UR                  40 UD640 [	        U5      D6$ r'  r(  r)  s         r   fill_Scalarr  	  r+  r%   c                 h    [        XUSS9u  p4UR                  S5      nU " UR                  40 UD6  U$ r'  rd  r)  s         r   fill__Scalarr  	  s;    &TMA ..
!C#
#Jr%   c                     [        XUSS9u  p4UR                  S5      n[        U5      nU " UR                  5      u  px[	        U40 UD6[	        U40 UD64$ r'  )r   r   r   r   r	   )	rg   rh   ri   r   r   r   ra  mantissaexponents	            r   frexp_Tensorr  
  si    &TMA ..
!C"3'Mckk*H2M2L5!5  r%   z+grad: any, self: any, other: any, mask: anyc                 |   [        XUSS9u  p4UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nUc  gS n	US   (       a&  [        R                  " XWR	                  S	S
5      5      n	S n
US   (       a'  [        R                  " UR	                  S	S
5      U5      n
X4$ )NTr   r  r   r  r  r   r   rC   r   )r   r   rH   r1  r  )rg   rh   ri   r   r   r  r   r  grad_input_mask	grad_self
grad_others              r   matmul_backward_defaultr  
  s    
 'TMA >>&!D
..
!CNN7#E nnV,O|IqLLr2'>?	Jq\\#--B"7>
""r%   )flex_attentionflex_attention_backward)GraphModulequerykeyr  	score_mod
block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc	                    U R                  5       S:X  a(  UR                  5       S:X  a  UR                  5       S:X  d   e[        S Xx-    5       5      (       a  [        S5      e[        U R	                  5       R                  S5      UR	                  5       R                  S5      UR	                  5       R                  S5      UUUUUUS9	n	[        R                  R                  U	S   R                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      n
[        R                  R                  U	S   R                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      nX4$ )	N   c              3   |   #    U  H2  n[        U[        R                  5      =(       a    UR                  v   M4     g 7fr   )r   rH   rI   r  )r   bufs     r   r   flex_njt.<locals>.<genexpr>C
  s.      CC 	3%7#--7Cs   :<zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r   )r  r  r  r  r  r  r   r   r   r   )r"   r6   r'   flex_attention_hopr  r  rH   r  nested_tensor_from_jaggedr  r   r   rP   r   r   )r  r  r  r  r  r  r  r  r  r$   
output_njtlogsumexp_njts               r   flex_njtr  4
  s    99;!	Q599;!3CCC  *C   U
 	
    #

q!  #% 75
F 77q	Aq!))!,**** 8  i1o  LL::q	Aq!))!,**** ;  i1o  &&r%   r  	logsumexpry  grad_logsumexpfw_graphjoint_graph.c                    [        U R                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UR                  5       R                  S5      UUU	U
UUUS9nUu  nnnn[        R                  R                  UR                  SS5      R                  S5      U R                  U R                  U R                  U R                  S9R                  SS5      n[        R                  R                  UR                  SS5      R                  S5      UR                  UR                  UR                  UR                  S9R                  SS5      n[        R                  R                  UR                  SS5      R                  S5      UR                  UR                  UR                  UR                  S9R                  SS5      nUUUU4$ )Nr   )r  r  ry  r  r  r  r  r  r  r  r  r   r   r  )flex_attention_backward_hopr  r  rH   r  r  r  r   r   rP   r   r   )r  r  r  r  r  ry  r  r  r  r  r  r  r  r  r$   dense_q_graddense_k_graddense_v_gradscore_mod_other_buffer_grads
njt_q_grad
njt_k_grad
njt_v_grads                         r   flex_njt_backwardr  m
  s   & )  #

q!  #JJL""1%""$..q1",,Q/%,,.88;% 75F$ NTJL,.J77q!$,,Q/**** 8  i1o  77q!$,,Q/(((( 8  i1o  77q!$,,Q/**** 8  i1o  
J0LMMr%   z	self: anyc                     SSK Jn  U" 5       $ )Nr   )_nt_view_dummy)$torch.nested._internal.nested_tensorr  )rg   rh   ri   r  s       r   _nested_get_jagged_dummyr  
  s    Cr%   r  IMPLr  CPUCUDAMeta)F)TFF)r   )r2   r2   )r   r  r   typingr   rH   torch.nn.functionalrl  rm  r  torch.fx.operator_schemasr   torch.nested._internal.sdpar   r   r	   r
   r   Any__annotations__r   Dictr   r/   r=   strrt   r|   r   r   r   r   register_jagged_funcCallabler   r   r   r   r  r  r  r  r  r	  r"   r  r
  r  r  r  primr@  r  r   r  rH  r  r  r3  r%  linearr*  linear_backwardr:  rH  r*  r<  _to_copyrT  rV  r^  r  
empty_like	ones_liker  	rand_like
randn_likerb  	full_likerandint_like	low_dtypezero_re  _softmaxrw  _softmax_backward_datarz  native_dropoutr  native_dropout_backwardr  r  dim_intr  r  r^   rI   r  r  r  r  r  r  r"  r  r  r   r  r  r  r  r  r1  r  bmmr  r   r  r  r  r  r  wherer/  r  _pin_memoryr  	is_pinnedr  is_same_sizer  r  r   r  rR  r  r  r  permuter  view_unsafe_viewr'  native_layer_normr:  native_layer_norm_backwardr?  rA  rB  rD  rE  	index_putrI  convolutionrP  r4  rU  rW  r6   r7   rY  r]  r   r`  rb  ri  rf  rd  rl  rr  rt  ru  rx  ry  r{  r|  r  r  r  r  r  r  	embeddingr  embedding_dense_backwardr  r  _nested_get_valuesr  r  r  r  _nested_from_padded_tensorr  _nested_view_from_jaggedr  r  r  r  r  r  r  r  _nested_select_backwardr  record_streamr  	new_empty	new_zerosnew_onesr  elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardr  fillScalarr  fill_r  frexpr  matmul_backwardr  &torch._higher_order_ops.flex_attentionr  r  r  r  torch.fx.graph_moduler  py_implTuplerp  r  Unionr  r  library_scoped_libraryimplr2   r%   r   <module>rC     s           8 K ' c #% $sCx. %3. :Y8=S =D =@

#&
+7
AD
	
0. !((8HI &HX,> &RKS\E$P 		33;;		''		""		$$		  ((		!!))		))11 00, eiinn++33^D E 
YY^^  !
	
 eiinn22::NKF LF( 	IINN  ..0S
 	IINN  "FEE2 eiinn++335UVP WP 	IINN""**<"	"J eiinn''--/JKP LP eiinn--55~F% G%P 	IINN  "Q: UYY^^**22N C
 		!!))		  ((		!!))		  ((		!!)) 	,5	,5^ UYY^^--557V W UYY^^0088:S T 	IINN))+N
 eiinn**22NC D 	IINN##%QEPEPP 	IINN))11=	 	IINN!!))+N

 	IINN**22+			 	IINN8>	> eiinn))113NO+ P+ 	IINN!G" 	IINN##++-T" 	IINN!!#P77  eiinn**224VW&
 X&
R eiinn++//1JKE LEP eiinn,,002FGK HK eiinn..668PQE RE, eiinn((002JK L0 eiinn++335LMj NjZ eiinn((002KLE ME  	IINN!!#LNN  eiinn..668LMK NK eiinn1199;TU V" eiinn66>>O P> 	IINNI$ eiinn0088:RSP TP eiinn..668PQ+ R+ 	IINN'')F**gFT eiinn((002MN+ O+ 	IINN""9=	= 	IINN  "F'P'PT eiinn,,446OP%G Q%GP 
YY^^  %)).."="="E"EF,R	,R^ 	IINN$$,,N^D	^DB 	IINN--55v
K	
K eiinn++//1UVG WG8 	IINN=
P	
P 	IINN$$> 	IINN%%>Q		Qh 	IINN&&G
P
P 	IINNR#L#LL eiinn))113NO+ P+ eiinn((--/WXA YA eiinn((,,.UV; W; eiinn((--/WX@ Y@ eiinn((,,.UV; W; 		""		""		""		""	 ++ eiinn((,,.UVE WE eiinn((,,.UVE WE 	IINN!IFF 	IINN!IFF 	IINN!!#KHH 	IINN!!#KHH 	IINN55==G=	=@ eiinn**224LM N< 	IINN$$W	 	IINN++33_D	D 		%%		))11 	 	  eiinn((00.A B 	IINN##++38	8v 	IINN--55i2	2j 	IINN++33i	8 eiinn88@@.Q R eiinn88@@.Q R eiinn;;CC^T U eiinn;;CC^T7 U7 eiinn;;CC^T7 U7 eiinn22::<QR S0 	IINN**228	 eiinn22::<RS# T# 		  ((		  ((		''
 Y
M
M 		##++		**22		++33		((00		((00		**22 
	
	 eiinn))002LMP NP eiinn**113MN O eiinn**11>B C 	IINN""**1#	#0 . L) &($&5'<<5'	5' <<5' 	5'
 5' 5' cN5' #5' "5' 5<<%&5' *5'p $$\2 &($&;N<<;N	;N <<;N 
	;N
 ||;N ll;N LL;N Hk)*;N ;N ;N ;N cN;N #;N ";N 	LL%,,eHU\\4JC4O.PP;N 3;N~ eiinn==EE{S T ]]""662dII(*BEJII(*BFKII(*BFK 322s   d0=Ae6e6
Af