
    [Th}             -       r   S SK r S SKJr  S SKJr  S SKJr  S SKJrJ	r	J
r
Jr  S SKJr  S SKrS SKJr  S SKJrJrJr  S SKJrJrJrJr  S S	KJr  S S
KJrJr  S SKJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(  S SK)J*r*J+r+J,r,J-r-J.r.  S SK/J0r0J1r1  S SK2J3r4  S SK5J6r7  \
" S5      r8\" S5      r9\Rt                  Rv                  r;\Rx                  R{                  SSS5      r>\?" S5      u  r@rArBS\\\9\84   /\\9\84   4   4S jrCS\#4S jrDS rES rF\C" \;R                  \;R                  /5      \." 5       SSS\R                  SS4S j5       5       rJ\C" \;R                  R                  \;R                  R                  /5      \." 5       S 5       5       rN\C" \;R                  R                  \;R                  R                  /5      \." 5       SS .S! j5       5       rO\C" \;R                  5      \." 5       S" 5       5       rP\C" \;R                  R                  \;R                  R                  \;R                  R                  \;R                  R                  /5      \." S#S$5      S% 5       5       rS\C" \;R                  R                  \;R                  R                  /5      \." 5       S& 5       5       rTS' rUGSS(\S)\V\W   S*\X4S+ jjrY\C" \;R                  R                  \;R                  R                  /5      \." 5       S, 5       5       r[Sr\S)\V\W   4S- jr]\C" \;R                  R                  \;R                  R                  /5      \." 5       S. 5       5       r_\C" \;R                  R                  5      SS/.S0 j5       rb\C" \;R                  R                  5      \R                  SSSS1.S2 j5       rd\C" \;R                  R                  \;R                  R                  /5      \." 5       \R                  SSSS1.S3 j5       5       rf\C" \;R                  R                  \;R                  R                  /5      \." 5       \R                  SSSS1.S4 j5       5       ri\C" \;R                  R                  \;R                  R                  /5      \." 5       SSSSS1.S5 j5       5       rk\C" \;R                  R                  \;R                  R                  /5      \." 5       S(\S)\V\W   S6\WS7\W4S8 j5       5       rm\C" \;R                  R                  5      GSS9 j5       roS: rp\C" \;R                  R                  5      S; 5       rr\C" \;R                  5         GSS<\S=\S>\S?\	\   S@\	\t   SA\	\R                     4SB jj5       rv\C" \;R                  5       GSSC\SD\SE\SA\	\R                     4SF jj5       rx\C" \;R                  5      SGSGSSH.S<\SC\SD\SE\SA\	\R                     4
SI jj5       rz\C" \;R                  5             GSSJ\R(                  SK\R(                  S?\	\   SL\	\   SA\	\R                     SM\XSN\WSO\WSP\X4SQ jj5       r|\C" \;R                  R                  5      SRSS.S(\S)\WST\SU\R(                  SV\tSW\XS\4SX jj5       r~\C" \;R                  R                  5      SRSS.S(\S)\WST\SU\R(                  SV\tSW\XS\4SY jj5       r\." 5       \C" \;GR                  R                  5      SZ 5       5       r\C" \;GR                  R                  5      SSSS SSS[.S\\SV\tS]\	\   S$\	\   S^\	\   S_\WS`\XS\4Sa jj5       r\C" \;GR
                  R                  \;GR
                  GR                  /5      \." 5       Sb 5       5       r\C" \;GR
                  GR                  5      GSSc j5       r\C" \;GR                  R                  \;GR                  GR                  /5      \." 5       Sd 5       5       r\C" \;GR                  GR                  5      GSSe j5       r\C" \;GR                  R                  5      Sf 5       r\C" \;GR                  R                  5      Sg 5       r\C" \;GR                   R                  5      Sh 5       r\C" \;GR                   GR$                  5      Si 5       r\C" \;GR(                  R                  5      Sj 5       r\C" \;GR,                  R                  5      SSSSSSk.Sl j5       r\C" \;GR0                  R                  5      GSSm j5       r\C" \;GR2                  R                  5      GSSn j5       r\C" \;GR6                  R                  5      GSSo j5       r\C" \;GR8                  R                  5      Sp 5       r\C" \;GR<                  GR$                  5      Sq 5       rS(\Sr\t4Ss jrS(\St\Su\t4Sv jr GSSw\Sr\tSx\X4Sy jjrGSSt\Sr\tSz\t4S{ jjrSt\S|\S}\XSr\t4S~ jr GSS\tS\S<\S\t4S jjrS\t4S jr\C" \;GRN                  R                  \;GRN                  GRP                  /5      \." SS5      GSSt\S\tS\X4S jj5       5       r\C" \;GRT                  R                  \;GRV                  R                  /5      \." 5       S<\S\4S j5       5       r\C" \;GRZ                  /5      \." SS5      S<\4S j5       5       rS\S\4S jr\C" \;GR`                  5      \." 5       S(\St\S\XS\4S j5       5       r\C" \;GRb                  5      \." 5       GSS(\St\S\XS\4S jj5       5       r\C" \;GRd                  5      \." 5       GSS(\S\XS\4S jj5       5       r\C" \;GRf                  5      \." 5       GSS(\S\XS\4S jj5       5       r\C" \;GRh                  R                  5      GSSt\S\XS\X4S jj5       r\C" \;GRj                  R                  \;GRj                  R                  /5      \." 5       S<\S\S\4S j5       5       r\C" \;GRl                  R                  5      GSSt\S\X4S jj5       r\C" \;GRp                  R                  \;GRp                  R                  /5      \." SSS5      SSS.S(\S\XS\XS\\\\4   4S jj5       5       r\C" \;GRv                  R                  \;GRv                  R                  /5      \." 5       SS.S\S\S|\S\XS\4
S jj5       5       r\C" \;GRz                  R                  \;GRz                  R                  /5      \." SSS5      SRS.St\S\XS\\\\4   4S jj5       5       r\C" \;GR~                  R                  \;GR~                  R                  /5      \." SSS5      SRSS.St\S\XS\XS\\\\4   4S jj5       5       r\C" \;GR                  R                  \;GR                  R                  /5      \." 5       SRSS.S\S\S|\S}\XS\XS\4S jj5       5       r\C" \;GR                  5      \." SSS5        GSS\S\S\XS\XS\\\\4   4
S jj5       5       rS\tS\\X\X4   4S jr\C" \;GR                  R                  \;GR                  R                  /5      \." SS5      GSSt\S\tS\\\4   4S jj5       5       r\C" \;GR                  R                  \;GR                  GR                  /5      \." SSSS5      St\S\\\\\4   4S j5       5       r\C" \;GR                  R                  5         GSSt\S\XS\XS\	\t   4S jj5       rS\S\S\\V\W   \V\W   4   4S jrS\S\Su\	\t   S\\\4   4S jrS<\S\S\X4S jr\C" \;GR                  5      SRSSSSSS.St\S|\S}\XS\XS\	\   S\	\   S\	\   S\	\   S\\\\\4   4S jj5       r\C" \;GR                  R                  \;GR                  R                  /5      SRSSS.St\S|\S\XS}\XS\XS\	\   S\4S jj5       r\C" \;GR                  5      \." SSSRS9   GSS(\St\S\XS\XS\XS\\\4   4S jj5       5       r\C" \;GR                  R                  5      S 5       r\C" \;GR                  5      \." 5         GSS<\S\S\S}\XS\XS\4S jj5       5       rS rS r\C" \;GR                  5      \." 5       S 5       5       r\C" \;GR                  5      \." 5       S 5       5       rS r\C" \;GR                  5      \." S5      S 5       5       r\C" \;GR                  5      \." S5      S 5       5       rS r\C" \;GR                  5      \." 5       S 5       5       r\C" \;GR                  5      \." 5       S 5       5       r\C" \;GR                  R                  \;GR                  GR                  \;GR                  R                  \;GR                  GR                  /5      \." S5      S 5       5       rS r\C" \;GR                  5      \." 5       S 5       5       r\C" \;GR                  5      \." 5       S 5       5       r\C" \;GR                  R                  \;GR                  GR                  \;GR                  R                  \;GR                  GR                  /5      \." S5      S 5       5       r\C" \;GR                  5      \." 5       GSS(\S\S\4S jj5       5       r\C" \;GR                  5      \." 5       S\S(\S\S\S\4
S j5       5       r\C" \;GR                  R                  \;GR                  R                  /5      \." 5       SGSGS.S j5       5       r\C" \;GR                  R                  \;GR                  R                  /5      \." 5       SS/.S j5       5       r\C" \;GR                  GR                  5      GS S j5       r\C" \;GR                  GR                  5      GS S j5       r\C" \;GR                   R                  \;GR                   R                  /5      \." 5       GSS j5       5       Gr\C" \;GR                  R                  5        GSS j5       Gr\C" \;GR                  5      \." 5       S 5       5       GrS GrGS!S jGr GSS\R(                  S=\R(                  S\\V\W   \W4   S\\V\W   \W4   S\\V\W   \W4   S\XS\WS\	\\V\W   \W4      4S jjGrS Gr	\C" \;GR                  R                  5      S\R(                  S=\R(                  S?\	\R(                     S\	\R(                     S\	\R(                     S\XS\S\4S j5       Gr\C" \;GR                  R                  5      S\R(                  S=\R(                  S?\R(                  S\V\W   S\V\W   S\V\W   S\XS\V\W   S\W4S j5       Gr\GR                  GR                  (       Gae  \Rx                  R{                  SSS5      Gr\C" \Rt                  GR"                  GR$                  R                  5      S 5       Gr\C" \Rt                  GR"                  GR(                  R                  5      S 5       Gr\GR                  GR,                  (       aN  \Rx                  R{                  SSS5      Gr\C" \Rt                  GR0                  GR2                  5      GS  5       Gr\Rx                  R{                  GSSS5      Gr\C" \Rt                  GR8                  GR:                  R                  5      GS 5       Gr\C" \Rt                  GR8                  GR:                  GR>                  5      GS 5       Gr \C" \Rt                  GR8                  GRB                  R                  5      \C" \Rt                  GR8                  GRB                  GRD                  5      GS 5       5       Gr#\C" \Rt                  GR8                  GRB                  GR>                  5      \C" \Rt                  GR8                  GRB                  GRH                  5      GS 5       5       Gr%\C" \Rt                  GR8                  GRL                  R                  5      \C" \Rt                  GR8                  GRN                  R                  5      GS 5       5       Gr(\Rx                  R{                  GSSS5      Gr)\C" \Rt                  GRT                  GRV                  5          GS"GS	 j5       Gr,\C" \Rt                  GRT                  GRZ                  5      GS
 5       Gr.GS Gr/\C" \;GR`                  R                  5           GS#GS j5       Gr1GS Gr2\C" \;GRf                  R                  5      GS 5       Gr4\C" \;GRj                  5      \." 5            GS#GS j5       5       Gr6\C" \;GRn                  5      \." S5      GS 5       5       Gr8\C" \;GRr                  R                  5      GS 5       Gr:\C" \;GRv                  R                  5      GS 5       Gr<\C" \;GRz                  R                  5      GS 5       Gr>\C" \;GR~                  5      \." S5      GS 5       5       Gr@GS\Sz\t4GS jGrA\C" \;GR                  5      \." SS$5      GS 5       5       GrC\C" \;GR                  5      \." S5      GS 5       5       GrE\C" \;GR                  5      \." SS$5      GS 5       5       GrG\C" \;GR                  5      \." S5      GS 5       5       GrI\C" \;GR                  R(                  5      GSGS j5       GrK\C" \;GR                  R                  \;GR                  R                  /5      \." 5       GS 5       5       GrM\C" \;GR                  R                  \;GR                  R                  /5      \." 5       SGS.GS\W4GS jj5       5       GrN\C" \Rt                  Rv                  GR                  R                  \Rt                  Rv                  GR                  R                  /5      \." 5       GS  5       5       GrO\C" \;GR                  R(                  \;GR                  R(                  /5      GS! 5       GrR\C" \;GR                  R                  /5      GS" 5       GrT\C" \;GR                  R                  \;GR                  R                  /5      \." 5       SGSGS.GS# j5       5       GrV\C" \;GR                  R                  \;GR                  R                  /5      SSGS$.GS% j5       GrY\C" \;GR                  R                  /5      SSGS$.GS& j5       Gr[\C" \;GR                  /5      \." 5       GS' 5       5       Gr]\C" \;GR                  /5      GS( 5       Gr_\C" \;GR                  /5      GS) 5       Gra\C" \;GR                  /5      GS* 5       Grc\C" \;GR                  /5      GS+ 5       GreGS,\WGS-\WS\W4GS. jGrfGS/ Grg\C" \;GR                  /5      S?\	\   4GS0 j5       Gri\C" \;GR                  /5      GS1 5       Grk\C" \;GR                  /5      GS2 5       Grm\C" \;GR                  R                  5      GS3 5       Gro\C" \;GR                  5      \." 5       GS4 5       5       Grq\C" \;GR                  R                  5            GS$GS5 j5       Grs\C" \;GR                  R                  5      GS6 5       GruGSGS7 jGrv\C" \;GR                  R                  \;GR                  R                  /5      \." 5       GS%SGS8.GS9 jj5       5       Grx\C" \;GR                  R                  \;GR                  R                  /5      GS: 5       Gr{\C" \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  \;GR                  R                  \;GR                  GR                  /5      \." S#S$5      GS&GS; j5       5       Gr\C" \;GR                   R                  5      GS< 5       Gr\C" \;GR                  R                  5      GS= 5       Gr\C" \;GR                  R                  5      GS> 5       Gr\C" \;GR                  GR                  \;GR                  GR                  \;GR                  R(                  \;GR                  R(                  \;GR                  R                  \;GR                  R                  \;GR                  R                  /5      GS? 5       Gr\C" \;GR                  GR                  \;GR                  GR                  \;GR                  R(                  \;GR                  R(                  /5      GSGS@ j5       Gr\C" \;GR                   R                  \;GR                   GR"                  /5      GSA 5       GrGSB Gr\C" \;GR(                  R(                  \;GR(                  GR                  /5      GSC 5       Gr\C" \;GR,                  R(                  \;GR,                  GR                  /5      GSD 5       Gr\C" \;GR0                  R                  5      GSE 5       Gr\C" \;GR4                  R(                  \;GR4                  GR                  /5      GSF 5       Gr\C" \;GR8                  R(                  \;GR8                  GR                  /5      GSG 5       Gr\C" \;GR<                  R                  5      GSH 5       Gr\C" \;GR@                  R(                  5      \." 5       GSS\4GSI jj5       5       Gr\C" \;GRD                  /5      \." 5        GS'GSJ j5       5       Gr\C" \;GRH                  /5       GS'GSK j5       Gr\C" \;GRL                  /5       GS'GSL j5       Gr\C" \;GRP                  R                  \;GRR                  R                  /5      GSGSM j5       Gr\C" \;GRV                  GR                  5      GSN 5       Gr\C" \;GRZ                  R                  5      GSO 5       Gr\C" \;GR^                  5      GSP 5       Gr\C" \;GRb                  5      \." 5       GSQ 5       5       Gr\C" \;GRf                  5      GSR 5       Gr\C" \;GRj                  R                  5      GSGSS j5       Gr\C" \;GRn                  R                  5      GST 5       GrGSGSU jGr\C" \;GRt                  R                  5      GSV 5       GrGSW GrGSX GrGSY GrGSZ Gr GSS<\GS[\WGS\\WGS]\WGS^\WGS_\WGS`\WGSa\WGSb\WGSc\WGSd\WGSe\WGSf\WGSg\WGSh\WGSi\WGSj\WGSk\WGSl\WGSm\WS\tGSn\X4,GSo jjGrGSp GrS<\GS\GS[\WGS\\WGS]\WGS^\WGS_\WGS`\WGSa\WGSb\WGSc\WGSd\WGSh\WGSi\WGSj\WGSk\WGSl\WGSm\WS\t4&GSq jGrGSr Gr\C" \;GR                  R                  5      GSs 5       Gr\C" \;GR                  R                  5          GS"GSt j5       Gr\C" \;GR                  R                  5      GSu 5       Gr\C" \;GR                  5      \." SS$5          GS"GSv j5       5       Gr\C" \;GR                  5      \." S5      GSw 5       5       GrS<\GSx\4GSy jGr " GSz GS{\5      GrS<\GSx\GS|\W4GS} jGr\C" \;GR                  R                  5      GS~ 5       Gr\C" \;GR                  5      \." 5       GS 5       5       Gr\C" \;GR                  5      \." SGS5      GS 5       5       Gr\C" \;GR                  R                  /5      GS 5       Gr\C" \;GR                  R                  5           GS(GS j5       Gr\C" \;GR                  R                  5      GS 5       Gr\C" \;GR                  R                  5      GS 5       Gr\C" \;GR                  R                  5      GS)GS j5       GrGSS)\WGS\WGS\X4GS jjGrGS GrGS Gr\C" \;GR                  R                  5      GSGS j5       GrGSGS jGrGSGS jGrGS GrGSGS jGrGS*GS jGr\C" \;GR                  R                  5      GS 5       Gr\C" \;GR                  5      GS 5       Gr\C" \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  /5      \." 5       GSGS j5       5       Gr\C" \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  \;GR                  GR                  /5      GSGS j5       Gr\C" \;GR                  /5          GS+GS\GS\GS\GS\GS\XGS\XGS\	\   4GS jj5       Gr\C" \;GR                  /5          GS+GS\GS\GS\GS\	\   GS\XGS\GS\XGS\XGS\	\   4GS jj5       Gr\C" \;GR                  /5           GS,GS\GS\GS\GS\	\   GS\GS\XGS\XGS\	\   4GS jj5       Gr\C" \;GR                  /5       GSGS\GS\GS\GS\S\GS\GS\GS\GS\WGS\WGS\GS\XGS\GS\GS\	\   4GS jj5       Gr\C" \;GR                  /5          GS-GS\GS\GS\GS\GS\XGS\	\   GS\	\   4GS jj5       Gr\C" \;GR                  /5        GSGS\GS\GS\GS\S\GS\GS\GS\XGS\	\   GS\	\   4GS jj5       Gr\C" \;GR                  /5         GS.GS\GS\GS\GS\	\   GS\XGS\XGS\	\   4GS jj5       Gr \C" \;GR                  /5        GS/GS\GS\GS\GS\GS\	\   S\GS\GS\GS\GS\GS\V\X   GS\XGS\	\   4GS jj5       Gr\C" \;GR                  /5       GSGS\GS\GS\GS\S\GS\GS\GS\GS\GS\GS\GS\WGS\WGS\GS\XGS\	\   4 GS jj5       Gr\C" \;GR
                  /5           GS(GS\GS\GS\GS\	\   GS\	\   GS\WGS\WGS\GS\XGS\XGS\	\   GS\	\W   GS\	\W   GS\	\   GS\	\   4GS jj5       Gr\C" \;GR                  /5         GSGS\GS\GS\GS\S\GS\GS\GS\GS\WGS\WGS\GS\XGS\GS\GS\	\   GS\	\W   GS\	\W   4"GS jj5       Gr\C" \;GR                  /5           GS0GS\GS\GS\S?\	\   GS\	\   GS\	\   GS\	\W   GS\	\W   GS\GS\WGS\XGS\	\   GS\	\   GS\	\   GS\	\W   4GS jj5       Gr
\C" \;GR                  /5         GS*GS\GS\GS\GS\S?\	\   GS\	\   GS\	\   GS\GR                  GS\GR                  GS\GS\GS\GS\GS\WGS\XGS\	\   GS\	\W   GS\X4$GS jj5       Gr\C" \;GR                  R                  /5          GS1S(\R(                  SE\R(                  GS\R(                  GS\R(                  S?\	\R(                     GS\	\R(                     SA\	\R                     GS\X4GS jj5       Gr\C" \;GR                   GR"                  \;GR                   GR$                  /5      \." 5       GSGS j5       5       Gr\C" \;GR(                  GR"                  5      GSGS j5       Gr\C" \;GR,                  R                  \;GR,                  R                  /5      \." 5       GSSS/.GS jj5       5       GrGS GrGS Gr\C" \;GR4                  R                  \;GR6                  R                  /5      GSGS j5       Gr\C" \;GR8                  R                  \;GR:                  R                  /5      GSGS j5       Gr\C" \;GR<                  R                  \;GR>                  R                  /5        GSGS\GS\\\W\GR                  4      GS\\\W\GR                  4      GS\	\   GS\	\   4
GS jj5       Gr\C" \;GR@                  R                  \;GRB                  R                  /5      GSGS j5       Gr \C" \;GRD                  R                  \;GRD                  GRF                  \;GRD                  GR                  \;GRD                  GRH                  /5      GS2GS j5       Gr%GS Gr&\C" \;GRN                  R                  5        GSGS j5       Gr(\C" \;GRR                  R                  5      GS 5       Gr)\C" \;GRT                  R                  5      GS 5       Gr*GS Gr+GS Gr,\C" \;GRZ                  R                  \;GR\                  R                  /5      GS%GS j5       Gr/\C" \;GR`                  R                  5      GS3GS j5       Gr0\C" \;GRb                  R                  5      GS4GS j5       Gr2\C" \;GRf                  5      \." 5        GS5GS j5       5       Gr4\C" \;GRj                  R                  \;GRj                  GR                  /5      \." S#S$5      GS&GS j5       5       Gr6\GRn                  Gr8GS Gr9\C" \;GRt                  R                  5      GS 5       Gr:\C" \;GRv                  R                  5      GS 5       Gr;\C" \;GRx                  R                  5      GS 5       Gr=\C" \;GR|                  R                  5      GS 5       Gr>\C" \;GR~                  R(                  \;GR~                  GR                  /5      \." 5       SSGS.GS j5       5       GrA\C" \;GR                  /5      \." 5       GS6GS j5       5       GrC\C" \;GR                  R                  \;GR                  R                  /5        GSGS j5       GrF\C" \;GR                  R                  /5        GSGS j5       GrH\C" \;GR                  R                  5      GS 5       GrI\C" \;GR                  R                  \;GR                  R                  /5      \." 5       GSGS j5       5       GrJ\C" \Rt                  Rv                  GR                  5      GS 5       GrK\C" \Rt                  Rv                  GR                  5      GS 5       GrL\C" \;GR                  5      \." 5       SSSSGS.GS j5       5       GrNGS GrO\C" \;GR                  5       GS7GS j5       GrQ\C" \;GR                  5       GS7GS j5       GrS\C" \;GR                  5       GS7GS j5       GrU\C" \;GR                  5      \." 5       SSGS.GS j5       5       GrW\C" \;GR                  5      \." 5       GS\WS(\S\4GS j5       5       GrY\C" \;GR                  5      S(\4GS j5       Gr[\C" \;GR                  5      \." SRS9S(\S\4GS j5       5       Gr\\C" \;GR                  5      \." 5       S(\S\4GS j5       5       Gr]\C" \;GR                  5      \." 5       GS\S)\WGS\XS\4GS  j5       5       Gr_\C" \;GR                  5      \." 5          GS8S=\S$\GS\WGS\XGS\XS\4GS jj5       5       Gr`\C" \;GR                  R                  5       GS9S#\S^\V\   GS\V\W   GS\4GS jj5       GrbGS GrcGS	 GrdG\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\c" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        G\d" \;GR                  5        GS
 Grz\C" \;GR                  5      \." 5       GS 5       5       Gr{\C" \;GR                  5      \." 5       SGGS.GS j5       5       Gr|\C" \;GR                  5      \." 5       SGGS.GS j5       5       Gr}G\z" \;GR                  5      Gr~G\z" \;GR                  5      GrG\z" \;GR                  5      GrS SK/rS SGKrS SGKrGS GrG\" 5         g(:      N)Sequence)Enum)wraps)CallableOptionalTypeVarUnion)	ParamSpec)SymBoolSymFloatTensor)_add_op_to_registry_convert_out_paramsglobal_decomposition_table
meta_table)
OpOverload)_prim_elementwise_meta$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)
BoolLikecorresponding_complex_dtypecorresponding_real_dtypeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KIND	FloatLikeIntLikemake_contiguous_strides_forNumber
TensorLike)_maybe_convert_to_dtype_maybe_resize_out_resize_output_check_safe_copy_outout_wrapper)_broadcast_shapes_maybe_broadcast)_config)_pytree_T_PatenIMPLMeta   returnc                    ^  U 4S jnU$ )Nc                 Z   >^  [        T 5      m U 4S jn[        R                  " UT5        T $ )Nc                 (   > [        [        U T5        g N)r   r   )opfns    Q/var/www/auris/envauris/lib/python3.13/site-packages/torch/_meta_registrations.pyregister0register_meta.<locals>.wrapper.<locals>.register9   s    
B3    )r   pytree	tree_map_)r4   r6   r3   s   ` r5   wrapperregister_meta.<locals>.wrapper6   s)     $	4 	2&	r8    )r3   r;   s   ` r5   register_metar>   5   s     Nr8   type_promotionc                     [         R                  " USU 06u  p#U Vs/ s H  n[        XC5      PM     nn[        U6 n[	        US[
        R                  06$ s  snf )Ntype_promotion_kindr?   )utilsr   r   r%   r   r   DEFAULT)r?   args_result_dtypexs        r5   elementwise_metarH   B   sp    
 ..	*OA ?CCd#A4dDC T"D "	BJJ  Ds   Ac                     [         R                  [         R                  [         R                  [         R                  [         R
                  [         R                  0nUR                  X 5      $ r2   )torch	complex32halfcfloatfloatcdoubledoubleget)dtypefrom_complexs     r5   toRealValueTyperT   V   sC    ekku||L
 E))r8   c                 p   ^ ^ [        [        T /UQ76 5      m[        R                  " TT :H  UU 4S j5        g )Nc                     > ST ST  3$ )Nzoutput with shape z# doesn't match the broadcast shape r=   )broadcasted_shape
self_shapes   r5   <lambda>)check_inplace_broadcast.<locals>.<lambda>c   s    $ZL0STeSfgr8   )tupler$   rJ   _check)rX   
args_shaperW   s   ` @r5   check_inplace_broadcastr^   _   s0    /
HZHI	LLZ'gr8   Fc	           	        ^ ^^^^	 [        T [        R                  5      (       a)  [        R                  " T R	                  5       S:H  S 5        [        T[        R                  5      (       a)  [        R                  " TR	                  5       S:H  S 5        [        S T TT4 5       5      (       a`  [        R                  " [        R                  " 5       5      m	Tc  T	mOO[        R                  " [        R                  " T5      U	U4S j5        OT=(       d    [        R                  " 5       m[        T[        R                  5      (       d   e[        R                  " [        T[        5      UU U4S j5        [        T[        5      (       d   e[        R                  " TS:  S 5        [        R                  " T4TUSUUS	9$ )
Nr   c                      gNz:linspace only supports 0-dimensional start and end tensorsr=   r=   r8   r5   rY   (meta_linspace_logspace.<locals>.<lambda>w       Pr8   c                      gra   r=   r=   r8   r5   rY   rb   |   rc   r8   c              3   B   #    U  H  n[        U[        5      v   M     g 7fr2   )
isinstancecomplex).0args     r5   	<genexpr>)meta_linspace_logspace.<locals>.<genexpr>   s     
C/B:c7##/Bs   c                     > ST  ST 3$ )Nzlinspace(): inferred dtype z& can't be safely cast to passed dtype r=   )default_complex_dtyperR   s   r5   rY   rb      s    56K5LLrsxryzr8   c                     > S[        T5      R                   S[        T 5      R                   S[        T5      R                   S3$ )Nz4received an invalid combination of arguments - got (, ))type__name__)endstartstepss   r5   rY   rb      sB     u+r$s),,-RU0D0D/EQHr8   c                      g)Nz$number of steps must be non-negativer=   r=   r8   r5   rY   rb      s    %Kr8   meta)rR   layoutdevice
pin_memoryrequires_grad)rf   rJ   r   r\   dimanyrB   r   get_default_dtypeis_complex_dtyperR   _check_typer   empty)
rt   rs   ru   baserR   ry   rx   rz   r{   rm   s
   ``` `    @r5   meta_linspace_logspacer   g   s[    %&&IIK1P	
 #u||$$GGINP	

 
CsE/B
CCC % A A##%!
 =)ELL&&u-z
 2002eU[[)))) 
5'"	H
 eW%%%%	LL!KL;;	# r8   c                 6  ^ [         R                  " TR                  [         R                  :H  U4S j5        [         R                  " U R                  5       S:H  =(       a    TR                  5       S:g  (       + S 5        U R                  TR                  5      $ )Nc                  "   > ST R                    3$ )Nz2take(): Expected a long tensor for index, but got rR   indexs   r5   rY   meta_take.<locals>.<lambda>   s    DU[[MRr8   r   c                      g)Nz*take(): tried to take from an empty tensorr=   r=   r8   r5   rY   r      s    <r8   )rJ   r\   rR   long_check_indexnumel	new_emptyshape)selfr   s    `r5   	meta_taker      sm     
LLuzz!R
 
ZZ\Q55;;=A#56< >>%++&&r8   r|   c                b  ^ ^^ T R                   nTR                   n[        R                  " X4:H  S 5        [        R                  " T R                  T5      S:H  =(       a    TR                  T5      S:H  UUU 4S j5        [	        T R
                  TR
                  5      nT R                  U5      $ )Nc                      g)Nz=linalg.cross: inputs must have the same number of dimensions.r=   r=   r8   r5   rY   linalg_cross.<locals>.<lambda>       Or8   r-   c                  V   > ST  STR                  T 5       STR                  T 5       3$ )Nzlinalg.cross: inputs dimension z must have length 3. Got  and size)r|   otherr   s   r5   rY   r      s1    -cU 399S>"%

3'8:r8   )ndimrJ   r\   r   r$   r   r   )r   r   r|   x_dy_d	out_shapes   ```   r5   linalg_crossr      s     ))C
**C	LL
O 
LL		#!4

31 4	
 "$**ekk:I>>)$$r8   c                 x    [        U S5        [        U S5        [        R                  " U [        R                  S9$ )Nzlinalg.matrix_expmemory_format)squareCheckInputscheckFloatingOrComplexrJ   
empty_likecontiguous_formatr   s    r5   linalg_matrix_expr      s3     d/04!45D0G0GHHr8   valuesindicesc                 R   [         R                  " U R                  U R                  U R                  S9n[         R                  " U R                  U R                  [         R
                  S9nU R                  5       S:w  a%  U R                  S:w  a  [        XR                  5        X#4$ )Nry   rR   r   )	rJ   r   r   ry   rR   int64r   r   maybe_wrap_dim)r   r|   r   r   s       r5   	cummaxminr      sl    
 [[DKKtzzJFkk$**T[[LGzz|qTYY!^sII&?r8   c                 r    [        XR                  5        [        R                  " U [        R                  S9$ Nr   )r   r   rJ   r   r   )r   r|   s     r5   logcumsumexpr      s)     3		"D0G0GHHr8   c                V  ^ UR                   n[        U5      nXV-
  n[        [        U5      5      n[        U5       V	s/ s H  n	SPM     n
n	U H  nSX'   M	     / / pU H0  nX   (       d  UR	                  U5        M  UR	                  U5        M2     X-   n[        U5      nUR                  5       mUS U nUR                  U4S jSS9  XUS  -   nUR                  U5      nS/[        UR                  US  5      -   nUR                  U5      nUR                  S5      nUUS'   [        U5      n[        [        U5      5       H  nX#U      UUS-   '   M     U R                  U[        R                  S9  [        U5       V	s/ s H  n	SPM     nn	SnUS-
  nUS:  a1  UU R                  S5      -  UUU   '   UX(U      -  nUS-  nUS:  a  M1  [        Xu5       H   nU R                  SUU-
  -   5      UUU   '   M"     U R                  UUU R                  5       5        U $ s  sn	f s  sn	f )	NFTc                    > TU    $ r2   r=   )rG   self_stridess    r5   rY   _exec_fft.<locals>.<lambda>   s	    <?r8   keyreverser   r      r   )r   lenlistrangeappendstridesortpermuter   reshaper   resize_rJ   r   as_strided_storage_offset)outr   	out_sizesr|   forwardr   signal_ndim
batch_dimsdim_permuterE   is_transformed_dimdleftright	batch_endtmpinputbatched_sizes
batch_sizebatched_out_sizesiout_stridesbatch_numelr   s                          @r5   	_exec_fftr      s4   99Dc(K#J uT{#K).t5A%5 $  b%!$KKNLLO	 
 ,KD	I;;=L
jy
!CHH*DH9IJ//KLL%E D4JK 899MMM-(EAJ!M!]+3s8_#,V#4!a%  KK!1H1HKI $Dk*k1kK*KQA
q&&1CJJqM&AKN#yQ00	Q q& :$&)jja*n1E&FKN# %OOI{C,>,>,@AJW 6@ +s   H!H&r   r|   exclude_lastc                    ^ [        U5      nU R                  5       mUS [        U5      [        U5      -
   R	                  U4S jS9  U$ )Nc                    > TU    $ r2   r=   )r   r   s    r5   rY   _sort_dims.<locals>.<lambda>!  s	    l1or8   )r   )r   r   r   intr   )r   r|   r   sorted_dimsr   s       @r5   
_sort_dimsr     sL    s)K;;=L6#k"S%667<<% =  r8   c                 
   [         R                  " U R                  R                  5        U(       d  U R	                  5       $ [        X5      nU R                  U R                  5       5      n[        XPU R                  5       XCS9$ )Nr   )	rJ   r\   rR   
is_complexcloner   r   r   r   )r   r|   normalizationr   r   r   s         r5   meta_fft_c2cr   (  s\     
LL&&'zz|T'K
..
%CS		[JJr8   c                 n    [        U 5      [        :  d!  [        U 5      S:  a  U S   S:X  a
  U S   S:X  a  gg)N   r   r   FT)r   cufft_max_ndimr   s    r5   use_optimized_cufft_pathr   7  s3    
3x. SX]s1v{s1vQR{r8   c                   ^ [         R                  " U R                  R                  5        [	        U R                  5       5      n[	        U5      nUS   nXF   S-  S-   n[	        U5      nXxU'   U(       a  XuU'   [        U 5      S:X  Gai  U R                  U[        R                  " U R                  5      S9n	U n
[        U5      (       a  [        XXQSS9  O[        U5      S:X  a  UOUn[        XX/SS9  [        U5      S:  a.  U R                  U[        R                  " U R                  5      S9n
US S nU(       au  XpU
R                  5       mUR                  U4S jSS	9  [        [         [        U5      5      nU[        U5      U-
  S  n[        XXSS9  US [        U5      U-
   nU(       a  Mu  U(       d7  U	R                  U5      XV   :w  a   U
R#                  U[         R$                  S
9  U
n	U	$ [        U 5      S:X  aC  ['        XSS9nU R                  U[        R                  " U R                  5      S9n[        XX\SS9$ U R                  U[        R                  " U R                  5      S9$ )Nr   r   r   cudar   Tr   c                    > TU    $ r2   r=   )r   stridess    r5   rY   meta_fft_r2c.<locals>.<lambda>d  s	    '!*r8   r   r   xpu)r   )rJ   r\   rR   is_floating_pointr   r   device_hintr   rB   r   r   r   r   r   r   minr   r   r   r   )r   r|   r   onesidedinput_sizesr   last_dimlast_dim_halfsizeonesided_sizesoutputworking_tensortarget_sizesr   max_dims	last_dimsr   r   s                   @r5   meta_fft_r2cr   >  s_    
LL--.tyy{#K[!I2wH#-2Q6+&N08/(4F"U>>tzzJ   
 #C((fidK ),CA9>LflJPTU3x!|!%U%F%Ftzz%R "0 "
 cr(K)7(//1  ,d !  ~s;/?@'K(88(C(EF	Nt **GC,<x,GH + {{8$	(;;&&y@W@W&X'	T	e	# >nnU>>tzzJ  
 IDII~~U>>tzzJ  
 	
r8   )	generatorc                D    [        U[        R                  " U /5      5      $ r2   )r    rJ   Size)nr  r   s      r5   meta_randpermr    s    S%**aS/22r8   rR   rx   ry   rz   c                .    [         R                  " XX#US9$ Nr  rJ   r   )r  rR   rx   ry   rz   s        r5   meta_randperm_defaultr
    s     ;;	v r8   c                t   ^ ^ Sm[         R                  " T T:  U U4S j5        [         R                  " XX4US9$ )Nr   c                     > ST ST  3$ Nz:random_ expects 'from' to be less than 'to', but got from=z >= to=r=   highlows   r5   rY   meta_randint.<locals>.<lambda>      LSEQXY]X^_r8   r  rJ   r\   r   )r  r   rR   rx   ry   rz   r  s   `     @r5   meta_randintr    s:     C	LLs
_ ;;&J r8   c                p   ^ ^ [         R                  " TT :  UU 4S j5        [         R                  " X#XEUS9$ )Nc                     > ST ST  3$ r  r=   r  s   r5   rY   "meta_randint_low.<locals>.<lambda>  r  r8   r  r  )r  r  r   rR   rx   ry   rz   s   ``     r5   meta_randint_lowr    s5     
LLs
_ ;;&J r8   c                .    [         R                  " XX#US9$ r  r	  )r   rR   rx   ry   rz   s        r5   meta_rand_defaultr    s     ;;&J r8   r   lastdimc                    [         R                  " U R                  R                  5        [	        U 5      S:X  a  [        U R                  5       5      nX4US   '   U R                  U[        U R                  5      S9n[        U5      (       a(  [        UU R                  [         R                  S9UUSS9$ [        U5      S:  a  [        XS S SU5      nOU R                  [         R                  S9n[        XVXAS   /SS9$ U n[        U5      S:  a  US S n[        XUSS9nUSS  n[        UR                  5       5      nX4US   '   U R                  U[        U R                  5      S9n	[        XXASS9$ )	Nr   r   r   r   Fr   r   r   )rJ   r\   rR   r   r   r   r   r   rT   r   r   r   r   r   r   )
r   r|   r   r  r   r   tempr   c2c_dimsr   s
             r5   meta_fft_c2rr    s^    
LL&&'4F"%	$#b'	1LM#C((

)@)@
A  3x!|#Dcr(Aw?zz0G0GzHV92wiOO s8a<3BxH NEbc(C&	$#b'nnYodjj.InJYUCCr8   c                 `   SSK Jn  U" U 5      (       d%  [        R                  " U 5      S:X  a  [	        S5      e[        U[        5      (       a`  UR                  X5      nU R                  5       UR                  5       :w  a-  [        R                  R                  X@R                  5       5        U $ )Nr   )free_unbacked_symbolsr   zQmore than one element of the written-to tensor refers to a single memory location)%torch.fx.experimental.symbolic_shapesr!  rJ   _debug_has_internal_overlapRuntimeErrorrf   r   tor   r*   expand_copydefault)r   srcnon_blockingr!  intermediates        r5   
meta_copy_r+    s     L "$''E,M,Md,SWX,X_
 	
 #vvvd199;,++--$$\99;?Kr8   c                     [        U R                  5       5      n[        U R                  5       5      nXR                  5       :  a  SOX!   X1   -  nUR	                  US5        UR	                  X5        X#4$ Nr   )r   r   r   r|   insert)tensorr|   result_sizesresult_strides
new_strides        r5   inferUnsqueezeGeometryr3    sf    &L&--/*NZZ\)|/@>CV/VJQ#*''r8   c                 z    [        XR                  5       S-   5      n[        X5      u  p#U R                  X#5        U $ r-  )r   r|   r3  r   )r   r|   g_sizes	g_stridess       r5   meta_unsqueeze_r7    s6    
hhj1n
-C/:GW(Kr8   r   weight_metabias_activation_opt	out_dtypec                 @   [        U R                  5      nUb+  UR                  S5      UR                  S5      :X  d   S5       eUR                  S5      U R                  S5      S-  :X  d   eUR                  S5      US'   [        U R                  5      S:X  d   S5       eSU R                  S5      4nUb9  U R                  [
        R                  :X  a  U[
        R                  :X  d   S5       eU R                  UUc  U R                  OUS9R                  Xg5      nU$ )	Nr   zoutput size mismatchr   r   r   z*we can only handle the squashed input case9out_dtype is only supported for i8i8->i32 linear operatorr   )
r   r   r   r   rR   rJ   int8int32r   
as_strided)	r   r8  r9  r:  r;  r<  output_sizestransposed_stridesr   s	            r5   meta_sparse_structured_linearrD    s    $L{{1~1-E/EE-;;q>UZZ^a////{{1~L u{{q N"NN UZZ]+{{ejj(Y%++-E 	
G	
E __&.ekkI   j2 
 Mr8   mat1	mat1_metamat2c                    [        U R                  5      S:X  d   e[        UR                  5      S:X  d   e[        UR                  5      S:X  d   eU R                  S5      UR                  S5      S-  :X  d   eU R                  S5      UR                  S5      /nUb9  UR                  [        R
                  :X  a  U[        R                  :X  d   S5       eUR                  UUc  UR                  OUS9nU$ )Nr   r   r   r>  r   r   r   r   rR   rJ   r?  r@  r   )rE  rF  rG  r<  rB  r   s         r5   meta_sparse_structured_mmrJ  :  s     tzz?ay1$$$tzz?a99Q<499Q<!++++IIaL$))A,/LzzUZZ'I,D 	
G	
D ^^%-djj9  F
 Mr8   r   )alphabetar<  c                   [        U R                  5      S:X  d   S5       e[        UR                  5      S:X  d   e[        UR                  5      S:X  d   e[        UR                  5      S:X  d   eU R                  S5      UR                  S5      :X  d   S5       eUR                  S5      UR                  S5      S-  :X  d   eUR                  S5      UR                  S5      /nUb9  UR                  [        R
                  :X  a  U[        R                  :X  d   S5       eUR                  UUc  UR                  OUS9nU$ )Nr   zEonly input broadcasted to columns of mat1 * mat2 product is supportedr   r   r>  r   rI  )	r   rE  rF  rG  rK  rL  r<  rB  r   s	            r5   meta_sparse_structured_addmmrN  S  s/    u{{q  O  tzz?ay1$$$tzz?a::a=DIIaL( O( 99Q<499Q<!++++IIaL$))A,/LzzUZZ'I,D 	
G	
D ^^%-djj9  F
 Mr8   compressed_Adense_BrK  transpose_resultalg_idsplit_ksplit_k_one_kernelc	                    UR                   [        R                  [        R                  [        R                  [        R
                  [        R                  1;   d   S5       eU R                   UR                   :X  d   S5       e[        UR                  5      S:X  d   S5       eU R                   [        R
                  [        R                  4;   n	U	(       a  SOSn
U	(       a  UR                  5       (       a   S5       eUR                  S5      nUR                  S	5      nU R                  5       S
-  X-  -  nUb  XR                  S5      :X  d   eUbP  U	(       aB  U[        R                  [        R                  [        R                  [        R                  1;   d   S5       eU(       a  X4OX4nUR                  XS9$ )Nz;_cslt_sparse_mm only supports fp16, bf16, int8, and fp8e4m3zinputs must have the same dtyper   z'_cslt_sparse_mm only supports 2d inputs
   	   z.dense input must be transposed for 8bit dtypesr   r      z\out_dtype is not supported for {compressed_A.dtype} x {dense_B.dtype} -> {out_dtype} matmul!r   )rR   rJ   float32float16bfloat16r?  float8_e4m3fnr   r   is_contiguousr   r   r@  r   )rO  rP  r:  rK  r<  rQ  rR  rS  rT  is_8bit_input_typecompression_factorkr  moutput_shapes                  r5   meta__cslt_sparse_mmrc  v  s    ==

  E EE  .Q0QQ.w}}"M$MM"%++

E<O<O/PP1q((** 	
<	
* 	QAQA					"(:(>?AIIaL   !iMMNNKK	4
 '
 	
 k	
 
 .A6A6L\;;r8   T)include_selfr   sourcereducerd  c                H    [         R                  " U [         R                  S9$ r   rJ   r   r   r   r|   r   re  rf  rd  s         r5   meta_index_reducerj    s     D0G0GHHr8   c                    U $ r2   r=   ri  s         r5   meta_index_reduce_rl    s	     Kr8   c                     [        U R                  5       5      nU R                  5       S:  a  UR                  5       X1'   U R	                  U5      $ Nr   )r   r   r|   r   r   )r   r|   r   result_sizes       r5   meta_index_selectrp    s>     tyy{#KxxzA~ ;;=>>+&&r8   )lengthsr   offsetsaxisunsafeinitialdatarq  rr  rs  rt  c                   ^ ^ Ub  [        S5      eUU 4S jnUb  U" UR                  5      $ Ub+  UR                  S S UR                  S   S-
  4-   n	U" U	5      $ [        S5      e)Nz?segment_reduce(): indices based reduction is not supported yet.c                    > [         R                  " U TR                  TS-   S  -   TR                  S[         R                  S9$ )Nr   rw   rR   ry   r   )rJ   r   r   rR   r   )lengths_shapers  rv  s    r5   segment_reduce_lengths_tensor:meta_segment_reduce.<locals>.segment_reduce_lengths_tensor  s>    {{DJJtaxz22**11	
 	
r8   r   r   z<segment_reduce(): Either lengths or offsets must be defined.)NotImplementedErrorr   r$  )
rv  rf  rq  r   rr  rs  rt  ru  r{  rz  s
   `    `    r5   meta_segment_reducer~    s|     !M
 	

 ,W]];; cr*gmmB.?!.C-EE,];;
U
VVr8   c                 $    U R                  S5      $ Nr=   r   r   s    r5   meta_maxr         >>"r8   c                     [         R                  " U R                  U45      n[        XU5      nU R	                  U5      U R	                  U[
        R                  S94$ Nr   rB   reduction_dimsr   _compute_reduction_shaper   rJ   r   r   r|   keepdimrb  s       r5   meta_max_dimr    P    


tzzC6
2C+Dw?L|$|5::6 r8   c                 $    U R                  S5      $ r  r  r   s    r5   meta_minr    r  r8   c                     [         R                  " U R                  U45      n[        XU5      nU R	                  U5      U R	                  U[
        R                  S94$ r  r  r  s       r5   meta_min_dimr    r  r8   c                     U R                  5       (       a  [        U R                  5      nO[        U [        R
                  S9u  p![        R                  " XS9$ NrA   r   )r   r   rR   r   r   INT_TO_FLOATrJ   r   )r   rF   rE   s      r5   
meta_angler    sH    /

;, ? L L
 D55r8   c                     [         R                  " XR                  5       U R                  5        UR	                  [         R
                  " U 5      5      $ r2   )rJ   _resize_output_r   ry   copy_angle)r   r   s     r5   meta_angle_outr    s4    	#yy{DKK899U[[&''r8   c                     g r2   r=   )vals    r5   assert_asyncr  !      
r8   c                     g r2   r=   )r  
assert_msgs     r5   assert_async_metar  &  r  r8   c                     g r2   r=   )ss    r5   
print_metar  +  r  r8   rR   rx   ry   rz   r   c                 ,    [         R                  " SSS9$ )Nr   rw   ry   r	  r  s        r5   make_dep_tokenr  0  s     ;;q((r8   c                 j    SSK Jn  [        U [        [        45      (       a  [        S5      eU" XUS9  g )Nr   )constrain_range'Constraining SymFloat or Symbool is nyir   max)r"  r  rf   r   r   
ValueError)r   r   r  r  s       r5   sym_constrain_ranger  <  s0     F$7+,,BCCDs+r8   c                 .    [         R                  XUS9  U$ Nr  )r*   r  r   r   r  	dep_tokens       r5   functional_sym_constrain_ranger  F  s    T4r8   c                 6   SSK Jn  Uc  Uc  [        R                  " U 5        g [	        U [
        [        45      (       a  [        S5      e[        U 5      [        L a7  Ub  [        R                  " X:  5        Ub  [        R                  " X:*  5        g U" XUS9  g )Nr   )_constrain_range_for_sizer  r  )r"  r  rJ   _check_is_sizerf   r   r   r  rq   r   r\   )r   r   r  r  s       r5   sym_constrain_range_for_sizer  L  s     P
{s{T"$7+,,BCCDzS?LL%?LL%d5r8   c                 .    [         R                  XUS9  U$ r  )r*   r  r  s       r5   'functional_sym_constrain_range_for_sizer  `  s    %%d%=r8   c                     U$ r2   r=   )r  r  r  s      r5   functional_assert_async_metar  f  s    r8   f_namec                     U R                  5       S:  d
   U S35       eU R                  S5      U R                  S5      :X  d.   U SU R                  S5       SU R                  S5       S35       eg )Nr   z3: The input tensor must have at least 2 dimensions.r   z5: A must be batches of square matrices, but they are  by 	 matrices)r|   r   )r   r  s     r5   r   r   l  s}    88:? (EF? 99R=DIIbM) (G		RTVZ[_[d[deg[hZiirs)r8   Anamec                   ^ ^^ [         R                  " T R                  TR                  :H  UU 4S j5        [         R                  " T R                  TR                  :H  UU 4S j5        [         R                  " TR	                  S5      TR	                  S5      :H  U4S j5        [         R                  " TR	                  S5      T R	                  S5      :H  UUU 4S j5        g )Nc                  >   > STR                    ST R                    S3$ )Nz:Expected b and A to be on the same device, but found b on z
 and A on 	 instead.r  r  r   s   r5   rY   (linearSolveCheckInputs.<locals>.<lambda>{  s     H{{m:ahhZy:r8   c                  >   > STR                    ST R                    S3$ )Nz=Expected b and A to have the same dtype, but found b of type z and A of type r  r   r  s   r5   rY   r    s     Kzzl/!'')=r8   r   r  c                  R   > ST R                  S5       ST R                  S5       S3$ )Nz3A must be batches of square matrices, but they are r  r  r   r  r   r  s   r5   rY   r    s+    FF2J<tAFF2J<yBr8   c                     > ST ST R                  S5       ST R                  S5       STR                  S5       STR                  S5       3
$ )NzIncompatible matrix sizes for z: each A matrix is r   r  z but each b matrix is r  r   )r  r  r   s   r5   rY   r    sM    ,TF 3D$TYYr]O4		"Hr8   )rJ   r\   ry   rR   r   )r   r  r  s   ```r5   linearSolveCheckInputsr  x  s    	LLqxx	
 
LL

agg	
 
LL	r
affRj 	
 
LL	r
diim#	
r8   tallow_low_precision_dtypesc                 h  ^^ U R                   m[        R                  " U R                  5       =(       d    U R	                  5       UU4S j5        U(       d\  [        R                  " T[        R
                  [        R                  [        R                  [        R                  4;   UU4S j5        g g )Nc                     > T ST  3$ )Nz<: Expected a floating point or complex tensor as input. Got r=   rR   r  s   r5   rY   (checkFloatingOrComplex.<locals>.<lambda>  s    6(VW\V]^r8   c                     > T ST  3$ )Nz*: Low precision dtypes not supported. Got r=   r  s   r5   rY   r    s    vhHPr8   )	rR   rJ   r\   r   r   rN   rP   rM   rO   )r  r  r  rR   s    ` @r5   r   r     sn    
 GGE	LL	/^ &ekk5<<u}}MMP	
 &r8   arg_namec                 b   ^^ [         R                  " U R                  5       S:  UU4S j5        g )Nr   c                     > T ST  S3$ )Nz: The input tensor z! must have at least 2 dimensions.r=   )r  r  s   r5   rY   checkIsMatrix.<locals>.<lambda>  s    6(-hZ7XYr8   )rJ   r\   r|   )r  r  r  s    ``r5   checkIsMatrixr    s    	LL	1Yr8   Br   c                   ^ ^^^ [        T T5        [        TT5        [        R                  " T(       a#  T R	                  S5      TR	                  S5      :H  O"T R	                  S5      TR	                  S5      :H  U UUU4S j5        g )Nr  r   c                     > T ST(       a  SOS ST R                  S5       ST R                  S5       STR                  S5       STR                  S5       S	3$ )
Nz2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (r  rG   r   r   rp   r   )r  r  r  r   s   r5   rY   #checkInputsSolver.<locals>.<lambda>  sV    hHxX.AaffRj\qvvbzl!AFF2J<qJr8   )r   r  rJ   r\   r   )r  r  r   r  s   ````r5   checkInputsSolverr    sY    a !V	LL$(r
affRj affRjAFF2J.F	
r8   resultfn_nameresult_namec                 v   ^ ^^^ [         R                  " TR                  TR                  :H  U UUU4S j5        g )Nc            	      L   > T  ST ST STR                    STR                    3	$ )Nz: Expected z5 and input tensors to be on the same device, but got z on z and input on r  )r  r   r  r  s   r5   rY   !checkSameDevice.<locals>.<lambda>  s0    i{;-/dm4nU\\NLr8   )rJ   r\   ry   )r  r  r   r  s   ````r5   checkSameDevicer    s&     
LL%	
r8   UPLOc                    ^  T R                  5       n[        R                  " [        T 5      S:H  =(       a    US:H  =(       d    US:H  U 4S j5        g )Nr   ULc                     > ST  3$ )Nz1Expected UPLO argument to be 'L' or 'U', but got r=   )r  s   r5   rY   checkUplo.<locals>.<lambda>  s    CD6Jr8   )upperrJ   r\   r   )r  UPLO_uppercases   ` r5   	checkUplor    s<    ZZ\N	LLD	QKNc1J^s5JJr8   eigenvalueseigenvectorsr  	compute_vc                 P   [        U S5        [        U5        [        U R                  5      nU(       a,  U R	                  U5      nUR                  U[        USS95        OU R	                  S/5      nUR                  5         U R	                  U[        U R                  5      S9nXT4$ )Nzlinalg.eighF	row_majorr   r   )
r   r  r   r   r   r   r   poprT   rR   )r  r  r  r   vecsvalss         r5   meta__linalg_eighr    s     a'dOME{{5! ;EU ST{{A3	IIK;;uOAGG$<;=D:r8   c                     [        U S5        [        R                  " U R                  5      (       a  U R                  O[        R                  " U R                  5      nU R                  U R                  S S US9$ )Nzlinalg.eigvalsr   r   r   rB   r   rR   r   r   r   )r   complex_dtypes     r5   meta__linalg_eigvalsr    sf     e-. !!%++.. 	..u{{; 
 ??5;;s+=?AAr8   c                 0   [        U S5        [        R                  " U R                  5      (       a  U R                  O[        R                  " U R                  5      nU R                  U R                  S S US9nU R                  U R                  US9nX#4$ )Nz
linalg.eigr   r   r  )r   r  r   vectorss       r5   meta_linalg_eigr    s     e\* !!%++.. 	..u{{; 
 __U[["-]_CFooekko?G?r8   r(  c                 p    U R                   R                  [        R                  S9R	                  SS5      $ )Nr   r  r   )mTr   rJ   r   	transpose)r(  s    r5   cloneBatchedColumnMajorr    s*    66<<e&=&=<>HHRPPr8   r  c                     [        U 5      $ r2   )r  )r   r  r  s      r5   _cholesky_solve_helperr  
  s     #4((r8   c                    ^ ^ [         R                  " T R                  S:  U 4S j5        [         R                  " TR                  S:  U4S j5        [        T TS5      u  p4[	        X4U5      $ )Nr   c                  $   > ST R                    S3$ )Nz-b should have at least 2 dimensions, but has  dimensions insteadr   r   s   r5   rY    cholesky_solve.<locals>.<lambda>  s    ?		{J]^r8   c                  $   > ST R                    S3$ )Nz-u should have at least 2 dimensions, but has r  r	  r  s   r5   rY   r
    s    ?xGZ[r8   cholesky_solve)rJ   r\   r   !_linalg_broadcast_batch_dims_namer  )r   r  r  self_broadcastedA_broadcasteds   ``   r5   r  r    sd     
LL		Q^ 
LL	![ 'Ha!'# ""25IIr8   c                     U R                  5       S:X  a#  [        R                  " U [        R                  S9$ [	        U S5        [        U 5      $ )Nr   r   cholesky)r   rJ   r   legacy_contiguous_formatr   r  r   r  s     r5   r  r  !  s@     zz|qE4R4RSSdJ'"4((r8   c                 0    [        U S5        [        U 5      $ )Ncholesky_inverse)r   r  r  s     r5   r  r  *  s     d./"4((r8   check_errorsc                 
   [        U S5        [        U S5        U R                  n[        U5      n[	        US5      nU R                  U5      nUR                  X55        U R                  USUS-
   [        R                  S9nXg4$ )Nzlinalg.choleskyFr   r   r   )	r   r   r   r   r   r   r   rJ   r@  )r  r  r  A_shaper   	L_stridesr  infoss           r5   linalg_cholesky_exr  2  s|    a*+1/0ggGw<D ,GU;I	GAMM'% KKD1H-U[[KAE8Or8   tauc                 J  ^ ^^ [         R                  " T R                  S:  S 5        [         R                  " T R                  S5      T R                  S5      :  S 5        [         R                  " T R                  S5      TR                  S5      :  S 5        [         R                  " T R                  TR                  -
  S:H  U U4S j5        T R                  S:  a<  T R                  S S nTR                  S S m[         R                  " TU:H  U4S	 j5        [         R                  " TR
                  T R
                  :H  U U4S
 j5        [        STT S5        [         R                  " T R                  [        T R                  SS9T R
                  T R                  S9$ )Nr   c                      g)NzHtorch.linalg.householder_product: input must have at least 2 dimensions.r=   r=   r8   r5   rY   ,linalg_householder_product.<locals>.<lambda>K      Zr8   r  r   c                      g)Nzbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r=   r=   r8   r5   rY   r  O  s    tr8   c                      g)Nz`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r=   r=   r8   r5   rY   r  S  s    rr8   r   c                  <   > STR                    ST R                    3$ )Nzptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to  and input.ndim is equal to r	  r   r  s   r5   rY   r  X  "    )),
2Nuzzl\r8   c                     > ST  3$ )Nzltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r=   actual_batch_tau_shapes   r5   rY   r  b      66L5MOr8   c                  <   > STR                    ST R                    3$ )Nz,torch.linalg.householder_product: tau dtype z does not match input dtype r   r%  s   r5   rY   r  j  s    :399+*5;;-9r8   z torch.linalg.householder_productr  Fr  r   r   rR   ry   )
rJ   r\   r   r   r   rR   r  empty_stridedr   ry   )r   r  expected_batch_tau_shaper)  s   `` @r5   linalg_householder_productr/  D  sK   
 
LL

aZ 
LL

2%**R.(t 
LL

2#((2,&r
 
LL

SXX"	
 zzA~#(;;s#3 !$3B"&>>	
 
LL		U[[ 	
 6UEJ[[*5;;%Hkk||	 r8   c                    [        U S5        [        U SSS9  U R                  U R                  5      nUR	                  U R                  [        U R                  SS95        U R                  U R                  S S [        R                  S9nX#4$ )Nzlinalg.inv_exF)r  r  r  r   r   r   r   r   r   r   rJ   r@  )r  r  r  r  s       r5   linalg_inv_ex_metar2  z  so    a)1o%P	AGGAMM!''6qww%PQKKEKKK8E8Or8   LDpivotsinfo)	hermitianr  r6  c                t   [        U S5        [        U S5        [        R                  " U R                  [        U R                  SS9U R                  U R                  S9nU R                  U R                  S S [        R                  S9nU R                  U R                  S S [        R                  S9nX4U4$ )Nztorch.linalg.ldl_factor_exFr  r,  r   r   r  )
r   r   rJ   r-  r   r   rR   ry   r   r   )r   r6  r  r3  r4  r5  s         r5   linalg_ldl_factor_ex_metar8    s     d894!=>			ZZ*4::Gjj{{	
B ^^DJJsO599^=F>>$**Sb/>;Dtr8   )r6  c                j  ^ ^^ [        T S5        [        T S5        [        TT S5        [        R                  " TR
                  S:  U4S j5        T R                  S S n[        R                  " UTR                  :H  U4S j5        [        R                  " [        R                  " TR                  5      U4S j5        [        R                  " T R                  TR                  :H  UU 4S j5        [        TT 5      u  pV[        R                  " U[        USS	9TR                  TR                  S
9$ )Nztorch.linalg.ldl_solver   c                  $   > ST R                    S3$ )NzMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has r  r	  )r  s   r5   rY   'linalg_ldl_solve_meta.<locals>.<lambda>      &&!46r8   r   c                  $   > ST R                    S3$ )Nzjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape  insteadr   r4  s   r5   rY   r;        ))/h@r8   c                  "   > ST R                    3$ )Nz<torch.linalg.ldl_solve: Expected pivots to be integers. Got r   r@  s   r5   rY   r;    s    Nv||n]r8   c                  <   > STR                    ST R                    3$ )Nz!torch.linalg.ldl_solve: LD dtype z does not match b dtype r   )r  r3  s   r5   rY   r;    s     3BHH:=UVWV]V]U^_r8   Fr  r,  )r   r   r  rJ   r\   r   r   rB   is_integer_dtyperR   _linalg_broadcast_batch_dimsr-  r   ry   )r3  r4  r  r6  expected_pivots_shapeB_broadcast_sizerE   s   ```    r5   linalg_ldl_solve_metarH    s     b232781b":;	LL	!	
 HHSbM	LL-	
 
LLv||,] 
LL
AGG_ 7q"=*+;uMggxx	 r8   Pr  )pivotrJ  c                j  ^  [         R                  " T R                  S:  U 4S j5        [        T R                  5      nUS   nUS   n[        X45      nX2S'   U(       a  T R                  U5      nOT R                  S/5      nXRS'   T R                  U5      nXRS'   XBS'   T R                  U5      nXgU4$ )Nr   c                  $   > ST R                    S3$ )Nz@linalg.lu: Expected tensor with 2 or more dimensions. Got size: r>  r?  r  s   r5   rY    linalg_lu_meta.<locals>.<lambda>  s    RSTSZSZR[[cdr8   r  r   r   )rJ   r\   r   r   r   r   r   )	r  rJ  sizesra  r  r`  rI  r  r  s	   `        r5   linalg_lu_metarO    s     
LL	!d
 MEb	Ab	AA	A"IKKKK"I	EA"I"I	EA7Nr8   LU)rJ  r  c                  ^  [         R                  " T R                  S:  U 4S j5        [        T R                  5      nUS   nUS   n[         R
                  " U[        USS9T R                  T R                  S9nUR                  5         [        XE5      US'   T R                  U[         R                  S9nUR                  5         T R                  U[         R                  S9nXgU4$ )	Nr   c                  $   > ST R                    S3$ )NzFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: r>  r?  r  s   r5   rY   *linalg_lu_factor_ex_meta.<locals>.<lambda>  s    XYZY`Y`Xaaijr8   r  r   Fr  r,  r   )rJ   r\   r   r   r   r-  r   rR   ry   r  r   r   r   )	r  rJ  r  rN  ra  r  rP  r4  r5  s	   `        r5   linalg_lu_factor_ex_metarT    s     
LL	!j
 MEb	Ab	A			*5EBggxx	
B 
IIKA	E"I[[eii[0F 
IIK;;uEII;.Dtr8   )r   adjointrU  c                
  ^ ^^ [        T S5        [        R                  " T R                  TR                  :H  UU 4S j5        [        R                  " TR                  [        R                  :H  S 5        [        T S5        [        T TUS5        [        R                  " T R                  S5      TR                  S5      :H  S 5        [        R                  " T R                  S S TR                  :H  U4S j5        [        TT 5      u  pV[        R                  " U[        XS(       + S9TR                  TR                  S	9nUR                  5       S
:w  a,  U(       d%  UR                  5       (       a  UR                  5       nU$ )Nztorch.linalg.lu_solvec                  >   > STR                    ST R                    S3$ )NzPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type  and B of type r>  r   )r  rP  s   r5   rY   &linalg_lu_solve_meta.<locals>.<lambda>  s#    $$&HH:_QWWIXOr8   c                      g)NzElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r=   r=   r8   r5   rY   rY    s    Wr8   zlinalg.lu_solver   c                      g)NzYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr=   r=   r8   r5   rY   rY  $  s    kr8   c                  $   > ST R                    S3$ )Nzclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape r>  r?  r@  s   r5   rY   rY  *  rA  r8   r  r,  r   )r   rJ   r\   rR   r   r   r  r   r   rE  r-  r   ry   r   r   conj)rP  r4  r  r   rU  rG  rE   r  s   ```     r5   linalg_lu_solve_metar^    s-    267	LL
AGG	
 
LL		!W b12b!T#45	LL
v{{2&k 
LL
"%	
 7q"=  *+;xPggxx	F ||~4[[]FMr8   unpack_dataunpack_pivotsc                 6  ^  [         R                  " T R                  S:  U 4S j5        U(       a3  [         R                  " UR                  [         R                  :H  S 5        [        T R                  5      nUS   nUS   n[        XV5      nXTS'   U(       a  T R                  U5      nOT R                  S/5      nU(       a/  XtS'   T R                  U5      n	XtS'   XdS'   T R                  U5      n
O$T R                  S/5      n	T R                  S/5      n
XU
4$ )Nr   c                  $   > ST R                    S3$ )NzFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: r>  r?  )rP  s   r5   rY    lu_unpack_meta.<locals>.<lambda>J  s    XY[YaYaXbbjkr8   c                      g)Nztorch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr=   r=   r8   r5   rY   rc  O  s    pr8   r  r   r   )	rJ   r\   r   rR   r@  r   r   r   r   )rP  r4  r_  r`  rN  ra  r  r`  rI  r  r  s   `          r5   lu_unpack_metare  @  s     
LL
1k LLEKK'	
 NEb	Ab	AA	A"ILLLL!b	LLb	b	LLLL!LL!7Nr8   modec                    ^  T S:X  a  SnSnX4$ T S:X  a  SnSnX4$ T S:X  a  SnSnX4$ [         R                  " SU 4S j5        WW4$ )NreducedTcompleteFrc                     > ST  S3$ )Nzqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r=   )rf  s   r5   rY    _parse_qr_mode.<locals>.<lambda>w  s    1$ 8N Or8   rJ   r\   )rf  	compute_qrh  s   `  r5   _parse_qr_modero  j  s    y	  
		  
	  		
 gr8   QRc                    [        U S5        [        U S5        [        U5      u  p#U R                  S   nU R                  S   n[	        XE5      nU(       aO  [        U R                  5      nU(       a  UOUUS'   U R                  U5      nUR                  U[        USS95        OU R                  S/5      n[        U R                  5      n	U(       d  U(       d  UOUU	S'   U R                  U	5      n
U
R                  U	[        U	SS95        X4$ )Nz	linalg.qrr  r   Fr  r   )	r  r   ro  r   r   r   r   r   r   )r  rf  rn  reduced_modera  r  r`  Q_shaperp  R_shaperq  s              r5   linalg_qr_metarv    s     ![!1k*,T2I	A	AA	Aqww-'aQKK 	g:7eTUKK 177mG#9!!GBK	GAMM'6w%PQ4Kr8   sign	logabsdetc                 t   [        U S5        [        U SS5        U R                  nU R                  US S 5      nU R                  US S [	        U R
                  5      S9n[        R                  " U[        US5      U R
                  U R                  S9nU R                  US S [        R                  S9nX#XE4$ )Nzlinalg.slogdetFr  r   r,  r   )r   r   r   r   rT   rR   rJ   r-  r   ry   r@  )r  r   rw  rx  rP  r4  s         r5   _linalg_slogdetrz    s     a)*1.6GGE;;uSbz"DE#2Joagg.FGI			*5%8ggxx	
B [[s5;;[7FB&&r8   full_matrices
compute_uvdriverc                 b   [        U S5        [        U S5        [        U R                  S S 5      nU R                  S   nU R                  S   n[	        XV5      nU(       a  XEU(       a  UOU/-   nU R                  U5      n	U	R                  U[        USS95        XA(       a  UOUU/-   n
U R                  U
5      n[        U 5      S:H  nUR                  U
[        XS95        O$U R                  S/5      n	U R                  S/5      nU R                  XG/-   [        U R                  5      S9nXU4$ )	Nz
linalg.svdr  r   Fr  r   r   r   )r  r   r   r   r   r   r   r   r   rT   rR   )r  r{  r|  r}  r   ra  r  r`  U_shaper  V_shapeVis_cudaSs                 r5   _linalg_svd_metar    s    !\"1l+aggcrl#J	A	AA	A11==KK 	g:7eTU]1==KK 
 a.F*	g:7VW KKKK 	
J$OAGG,DEA7Nr8   arg1arg2c                    U R                   S S nUR                   S S n[        X#5      n[        U5      nXPR                  S5      U R                  S5      /-  n[        U5      nXaR                  S5      UR                  S5      /-  nXV4$ )Nr  r   )r   r$   r   r   )r  r  arg1_batch_sizesarg2_batch_sizesexpand_batch_portionarg1_expand_sizearg2_expand_sizes          r5   rE  rE    s    
 zz#2zz#2,-=P012		"66012		"66--r8   c                     U(       a  [        XU5        [        X5      u  p4X0R                  :X  a  U OU R                  U5      nXAR                  :X  a  UOUR                  U5      nXV4$ r2   )r  rE  r   expand)r  r  r  r  r  arg1_broadcastedarg2_broadcasteds          r5   r  r    sh     t40)Ed)Q& !JJ.DKK@P4Q  !JJ.DKK@P4Q  --r8   r   c                     U R                   S S nUR                  S:H  =(       d2    U R                  S-
  UR                  :H  =(       a    UR                   U:H  nU$ )Nr   r   )r   r   )r   r   expected_batched_rhs_shapevector_cases       r5   linalg_solve_is_vector_rhsr    sS    !&Sb!1**/ 

Q%**$R8R)R  r8   )r   r  r  rP  r4  r5  c                  ^ ^ [        T S5        [        R                  " T R                  TR                  :H  U U4S j5        [	        T T5      nU(       a  TR                  S5      OTn	[        T XS5        [        U	T 5      u  p[        R                  " U=(       d    U(       + S 5        U(       a  U
S S OU
n[        R                  " U[        X(       + 5      TR                  TR                  S9nT R                  n[        R                  " U[        US5      T R                  T R                  S9nT R                  US S [        R                  S9nT R                  US S [        R                  S9nXEXg4nXUU4n[        S	 U 5       5      (       aa  [        UU5       HQ  u  nn[!        UUR                  5        UR#                  UR                  UR%                  5       5        ['        UUSS
9  MS     U$ )Nzlinalg.solvec                  >   > ST R                    STR                    S3$ )NzKlinalg.solve: Expected A and B to have the same dtype, but found A of type rX  r>  r   )r  r  s   r5   rY   "_linalg_solve_ex.<locals>.<lambda>  s     Ywwiqwwix9r8   r   c                      g)Nzlinalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r=   r=   r8   r5   rY   r    s    Kr8   r,  Fr   r  c              3   (   #    U  H  oS Lv   M
     g 7fr2   r=   rh   rG   s     r5   rj   #_linalg_solve_ex.<locals>.<genexpr>3  s     
&#QD=#s   )	copy_fromcopy_toexact_dtype)r   rJ   r\   rR   r  	unsqueezer  rE  r-  r   ry   r   r   r@  allzipr    r   r   r"   )r  r  r   r  r  rP  r4  r5  r  B_B_broad_shaperE   result_shaperesult_r   LU_pivots_info_r   resrj  os   ``                    r5   _linalg_solve_exr    s    1n-	LL	177	
 -Q2K'RQBa>23B:M	LLK	
 *5="%-L!!*<Bggxx	G GGE


*5%8ggxx	C kk%*EKKk8GKKcr
%++K6Ev
$C%
(C

&#
&&&SMDAqa)MM!''188:.QuE " Jr8   )r   unitriangularr   r  r   c                   Uc  U R                  S/5      n[        U[        5      (       d   e[        XUS5        [	        XS 5      u  pgUR                  SS5      R                  5       =(       a    UR                  5       nU(       a  [        XVR                  5      nU$ [        XVR                  5      (       a=  UR                  UR                  SS5      R                  5        UR                  SS5        U$ )Nr   zlinalg.solve_triangularr  r   )r   rf   r   r  r  r  r]  is_conjr    r   r!   r   
transpose_)	r  r  r  r   r  r   r  A_avoid_copy_As	            r5   linalg_solve_triangular_metar  =  s     {kk1#c:&&&&aD";<.qT:FB<<B'557HBJJLLXX. J  XX..KKR,223NN2r"Jr8   XM)r  r  c                   ^ ^ [         R                  " T R                  S:  U 4S j5        [         R                  " TR                  S:  U4S j5        [        T TS5        TR                  [         R
                  :X  aw  [        T T5      u  pV[         R                  " U[        USS9T R                  T R                  S9n[         R                  " U[        USS9TR                  TR                  S9nXx4$ TR                  [         R                  :X  d  TR                  [         R                  :X  a+  [         R                  " T 5      nT R                  S/5      nXx4$ [         R                  " SS	 5        WW4$ )
Nr   c                  $   > ST R                    S3$ )NzMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has r  r	  r   s   r5   rY   'triangular_solve_meta.<locals>.<lambda>b  s    ))$79r8   c                  $   > ST R                    S3$ )NzMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has r  r	  r  s   r5   rY   r  i  r<  r8   triangular_solveFr  r,  r   c                      g)Nz+triangular_solve: Got an unexpected layout.r=   r=   r8   r5   rY   r    s    $Qr8   )rJ   r\   r   r  rx   stridedrE  r-  r   rR   ry   
sparse_csr
sparse_bsrr   r   )	r   r  r  r  r  self_broadcast_sizeA_broadcast_sizesolutioncloned_coefficients	   ``       r5   triangular_solve_metar  W  sC    
LL		Q	
 
LL	!	
 4$67xx5== 0LTST0U-&&$./BeT**;;	
 #00!./?5Q''88	
 '' 
U%%	%U5E5E)E##D)!^^QC0 '' 	UQR'''r8   c                 ^   [        U S5        [        U S5        U R                  U R                  S S 5      nU R                  U R                  5      nUR	                  U R                  [        U R                  SS95        U R                  U R                  S S [        R                  S9nXU4$ )Nz
linalg.detr  Fr  r   r   r1  )r  detrP  r4  s       r5   _linalg_det_metar    s    a&1l+
++aggcrl
#C	
QWW	BNN17775QR[["U[[[9FF?r8   c                 X  ^ ^^^^^ [         R                  " T R                  S:  S 5        [         R                  " TR                  S:  S 5        U(       a  SOSm[         R                  " TR                  T   TR                  S   :  U4S j5        [         R                  " TR                  T   T R                  S   :H  U4S j5        [         R                  " TR                  S   T R                  S   :*  S 5        [         R                  " T R                  TR                  -
  S	:H  U U4S
 j5        [         R                  " T R                  TR                  :H  U U4S j5        T R                  S:  ai  T R                  S S nTR                  S S m[         R                  " TU:H  U4S j5        TR                  S S m[         R                  " TU:H  U4S j5        [         R                  " TR                  T R                  :H  U U4S j5        [         R                  " TR                  T R                  :H  U U4S j5        [        STT S5        [        STT S5        [         R                  " TR                  [        TR                  SS9TR                  TR                  S9$ )Nr   c                      g)Nz3torch.ormqr: input must have at least 2 dimensions.r=   r=   r8   r5   rY   ormqr.<locals>.<lambda>      !Vr8   c                      g)Nz3torch.ormqr: other must have at least 2 dimensions.r=   r=   r8   r5   rY   r    r  r8   r  r   c                     > ST  S3$ )Ntorch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r=   left_size_conditions   r5   rY   r    s    +,?+@@pqr8   c                     > ST  S3$ )Nr  z"] must be equal to input.shape[-2]r=   r  s   r5   rY   r    s    +,?+@@bcr8   c                      g)NzHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r=   r=   r8   r5   rY   r    r   r8   r   c                  <   > STR                    ST R                    3$ )Nz[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to r$  r	  r%  s   r5   rY   r    r&  r8   c                  <   > STR                    ST R                    3$ )Nzhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to r$  r	  r   r   s   r5   rY   r    s&    ++0::,6RSXS]S]R^`r8   c                     > ST  3$ )NzWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r=   r(  s   r5   rY   r    r*  r8   c                     > ST  3$ )NzYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r=   )actual_batch_other_shapes   r5   rY   r    s    66N5OQr8   c                  <   > ST R                    STR                    3$ )NzPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype r   r%  s   r5   rY   r    s"    ##(;;-/B399+Or8   c                  <   > ST R                    STR                    3$ )NzRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype r   r  s   r5   rY   r    s"    ##(;;-/DU[[MSr8   ztorch.ormqrr  r   Fr  r,  )	rJ   r\   r   r   rR   r  r-  r   ry   )	r   r  r   r   r  expected_batch_shaper  r)  r  s	   ```   @@@r5   ormqrr    s    
LL

aV 
LL

aV !%""	LL'(CIIbM9q 
LL'(EKKO;c
 
LL		"R(Z
 
LL

SXX"	
 
LL

ejj 	
 zzA~${{3B/!$3B"&::	
 $);;s#3 $(<<	
 
LL		U[[ 	
 
LLu{{"	
 M3u5M5%9[[*5;;%Hkk||	 r8   c                  ^ ^^ [         R                  " [        T5      ST-  :H  UU4S j5        T R                  nUTS-   :H  nUnU(       + nU(       a1  [	        SU5       H   nU=(       a    T R                  U5      S:g  nM"     O0[	        SU5       H   nU=(       a    T R                  U5      S:g  nM"     [         R                  " U=(       d    UUU 4S j5        g )Nr   c                  ,   > SST -   S[        T5       3$ )Nzpadding size is expected to be r   z, but got: r   )r|   paddings   r5   rY   ,_padding_check_valid_input.<locals>.<lambda>  s    1!c'+c'l^Tr8   r   r   c                  :   > ST S-    ST S-    STR                    3$ )Nz	Expected r   zD or r   zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: r?  )r|   r   s   r5   rY   r  
  s-    aycAgY /AAFOr8   )rJ   r\   r   r   r   r   )r   r  r|   	input_dimis_batch_modevalid_batch_modevalid_non_batch_moder   s   ```     r5   _padding_check_valid_inputr    s    	LLGCT
 

I#'*M$,,q)$A/FEJJqMQ4F % q)$A#7#NEJJqMQ<N  % 
LL00	
r8   c                  ^ ^^^^	^
 SnSmSnT R                   S:X  a  T R                  S5      nTS-  mUS-  n[        T USS9  Uu  m	m
T R                  U5      nT R                  T5      mTT	-   T
-   mU(       a-  [        R                  " T	T:  =(       a    T
T:  UU U	U
4S j5        [        R                  " TS:  UU4S j5        T R                   S:X  a  T R                  UT45      $ T R                  XET45      $ )Nr   r   r-   r   c                  4   > ST ST ST  STR                    3$ NzcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (ro   ) at dimension 
 of input r?  dim_wr   pad_lpad_rs   r5   rY   _pad1d_common.<locals>.<lambda>&  -    %%*G2eWOE7*UZU`U`Tacr8   c                     > ST  ST 3$ )Nz
input (W: z%) is too small. Calculated output W: r=   )input_woutput_ws   r5   rY   r  .  s    *WI%J8*Ur8   r   )r   r   r  rJ   r\   r   )r   r  is_reflection	dim_planenbatchnplaner  r  r  r  r  s   `     @@@@@r5   _pad1d_commonr    s    IEFzzQA
Q	ug15LE5ZZ	"FjjG&HGO/	
 
LLAU
 zzQ1229::r8   c                     [        XSS9$ NTr  r  r   r  s     r5   meta_reflection_pad1dr  7       t<<r8   c                     [        XSS9$ NFr  r  r  s     r5   meta_replication_pad1dr  =       u==r8   c                  ^ ^^^^^ SmU(       d$  [         R                  " [        U5      S:H  S 5        TR                  S:X  a  TS-  mUu  mmTR	                  T5      nUT-   T-   mU(       a-  [         R                  " TU:  =(       a    TU:  UUUU4S j5        [         R                  " TT R	                  T5      :H  UU U4S j5        TR                  TR                  5      $ )Nr   r   c                      g)Nz padding size is expected to be 2r=   r=   r8   r5   rY   (_pad1d_backward_common.<locals>.<lambda>F  s    0Rr8   r-   c                  4   > ST ST ST  STR                    3$ r  r?  r  s   r5   rY   r  S  r  r8   c                  2   > ST STR                  T 5       3$ Nz(grad_output width unexpected. Expected: , Got: r   r  grad_outputr  s   r5   rY   r  [       :8*GKL\L\]bLcKder8   rJ   r\   r   r   r   r   r   )	r  r   r  r  r  r  r  r  r  s	   ``   @@@@r5   _pad1d_backward_commonr  C  s    ES\Q&(RSzzQ
LE5jjG&HGO/	
 
LLK$$U++e
 ??5;;''r8   
grad_inputc                     [        XUSS9$ r  r  r  r   r  s      r5   meta_reflection_pad1d_backwardr  a  s     "+gTRRr8   c                     [        XUSS9$ r  r  r  s      r5   meta_replication_pad1d_backwardr  g  s     "+gUSSr8   c                  ^ ^^^	^
^^^^^^ SmSmSnSn[        T USS9  T R                  nUS:X  a   T R                  S5      nTS-  mTS-  mUS-  nUu  mmmmT R                  U5      nT R                  T5      m	T R                  T5      m
T	T-   T-   mT
T-   T-   mU(       aZ  [        R                  " TT
:  =(       a    TT
:  UU UU4S j5        [        R                  " TT	:  =(       a    TT	:  UU UU4S j5        [        R                  " TS:  =(       d    TS:  U	U
UU4S j5        T R                  S	:X  a  T R                  UTT45      $ T R                  XFTT45      $ )
Nr   r   r   r      c                  4   > ST ST ST  STR                    3$ r  r?  r  s   r5   rY   _pad2d_common.<locals>.<lambda>  r  r8   c                  4   > ST ST ST  STR                    3$ NzcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (ro   r  r  r?  dim_hr   pad_bpad_ts   r5   rY   r    r  r8   c                      > ST  ST ST ST 3$ )Nz
input (H:  W: z%) is too small. Calculated output H: r=   )input_hr  output_hr  s   r5   rY   r    s%    	gY /$$,:T(=r8   r-   r  r   r   rJ   r\   r   )r   r  r  
dim_slicesr  r   r  r  r  r#  r  r$  r  r  r  r  r   s   `      @@@@@@@@@@r5   _pad2d_commonr'  m  sS   EEJFug15::DqyA

a
!(E5%ZZ
#FjjGjjG&H&HGO/	
 	GO/	
 
LLA&Q	
 zzQ(;<<(CDDr8   c                     [        XSS9$ r  r'  r  s     r5   meta_reflection_pad2dr*    r  r8   c                     [        XSS9$ r  r)  r  s     r5   meta_replication_pad2dr,    r  r8   c                   ^ ^^^^ SmSmSnUR                   nUR                  5       S:X  a  TS-  mTS-  mUS-  nUu  pVpxUT   n	UT   n
X-   U-   mX-   U-   m[        R                  " TT R	                  T5      :H  UU U4S j5        [        R                  " TT R	                  T5      :H  UU U4S j5        UR                  UR                   5      $ )Nr   r   r   r  c                  2   > ST STR                  T 5       3$ r	  r   r  s   r5   rY   %meta_pad2d_backward.<locals>.<lambda>  r  r8   c                  2   > ST STR                  T 5       3$ Nz)grad_output height unexpected. Expected: r
  r   r  r  r$  s   r5   rY   r/         ;H:W[M]M]^cMdLefr8   )r   r|   rJ   r\   r   r   )r  r   r  r  rX   r  r  r   r  r#  r  r  r  r$  r  s   `          @@@@r5   meta_pad2d_backwardr4    s     EEIJxxzQ

Q	!(E%GG&H&H	LLK$$U++e 
LLK$$U++f >>$**%%r8   c          	      ~  ^ ^^^	^
^^^^^^^^^^^ Sm	SmSmSn[        T USS9  T R                  S:H  nU(       a%  T R                  S5      nT	S-  m	TS-  mTS-  mUS-  nUu  mmmmmmT R                  U5      nT R                  T5      m
T R                  T5      mT R                  T	5      mT
T-   T-   mTT-   T-   mTT-   T-   mU(       a  [        R                  " TT:  =(       a    TT:  U	U UU4S j5        [        R                  " TT:  =(       a    TT:  UU UU4S j5        [        R                  " TT
:  =(       a    TT
:  UU UU4S	 j5        [        R                  " TS:  =(       d    TS:  =(       d    TS:  U
UUUUU4S
 j5        U(       a  T R                  WUTTT45      $ T R                  UTTT45      $ )Nr-   r   r   r   r      c                  4   > ST ST ST  STR                    3$ r  r?  r  s   r5   rY   _pad3d_common.<locals>.<lambda>  r  r8   c                  4   > ST ST ST  STR                    3$ r  r?  r  s   r5   rY   r8    r  r8   c                  4   > ST ST ST  STR                    3$ )NzcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (ro   r  r  r?  )dim_dr   pad_bkpad_fs   r5   rY   r8    s-    %%*G2fX_UG:V[VaVaUbdr8   c                  ,   > ST  ST ST ST ST ST 3$ )Nz
input (D:  H: r"  z%) is too small. Calculated output D: r=   )input_dr#  r  output_dr$  r  s   r5   rY   r8    s2    	gYd7) <$$,:T(4zKr8   r%  )r   r  r  r  
batch_moder  r  r;  r  r  r@  r#  r  rA  r$  r  r  r<  r=  r  r  r   s   `      @@@@@@@@@@@@@@@r5   _pad3d_commonrC    s   EEEIug15qJA


Q	07-E5%vZZ	"FjjGjjGjjG'H&H&HGO/	
 	GO/	
 	GO0 0	
 
LLA7Q7(a-	
 	
 (HMNN(HEFFr8   c                     [        XSS9$ r  rC  r  s     r5   meta_reflection_pad3drF    r  r8   c                     [        XSS9$ r  rE  r  s     r5   meta_replication_pad3drH    r  r8   c                   ^ ^^^^^^ [         R                  " [        U5      S:H  S 5        UR                  S:  d   eT R                  UR                  :X  d   eSmSmSmUR                  S:X  a  TS-  mTS-  mTS-  mUu  p4pVpxUR	                  T5      n	UR	                  T5      n
UR	                  T5      nX-   U-   mX-   U-   mX-   U-   m[         R                  " TT R	                  T5      :H  UU U4S j5        [         R                  " TT R	                  T5      :H  UU U4S j5        [         R                  " TT R	                  T5      :H  UU U4S	 j5        UR                  UR                  5      $ )
N   c                      g)Nz padding size is expected to be 6r=   r=   r8   r5   rY   %meta_pad3d_backward.<locals>.<lambda>)  s    ,Nr8   r-   r   r   r6  c                  2   > ST STR                  T 5       3$ r	  r   r  s   r5   rY   rL  A  r  r8   c                  2   > ST STR                  T 5       3$ r1  r   r2  s   r5   rY   rL  E  r3  r8   c                  2   > ST STR                  T 5       3$ )Nz(grad_output depth unexpected. Expected: r
  r   )r;  r  rA  s   r5   rY   rL  I  r  r8   r  )r  r   r  r  r  r   r  r=  r<  r@  r#  r  r;  r  r  rA  r$  r  s   `           @@@@@@r5   meta_pad3d_backwardrP    sS    
LLW"$NO::>>uzz)))EEEzzQ


07-E%jjGjjGjjG'H&H&H	LLK$$U++e 
LLK$$U++f 
LLK$$U++e
 ??5;;''r8   pc                 @   [         R                  " U R                  5       S 5        U R                  S5      nUS::  a-  U R	                  S/5      R                  [         R                  S9$ U R	                  X"S-
  -  S-  45      R                  [         R                  S9$ )Nc                      g)Nz(_pdist_forward requires contiguous inputr=   r=   r8   r5   rY   %meta__pdist_forward.<locals>.<lambda>S  s    &Pr8   r   r   r   r   )rJ   r\   r]  r   r   r%  r  )r   rQ  r  s      r5   meta__pdist_forwardrU  O  s     
LLP 			!AAv~~qc"%%E4R4R%SS~~qE{a/125588 6 
 	
r8   gradpdistc                     [         R                  " UR                  5       S 5        [         R                  " UR                  5       S 5        [         R                  " U[         R                  S9$ )Nc                      g)Nz._pdist_backward requires self to be contiguousr=   r=   r8   r5   rY   &meta__pdist_backward.<locals>.<lambda>b  s    &Vr8   c                      g)Nz/_pdist_backward requires pdist to be contiguousr=   r=   r8   r5   rY   rZ  e  s    'Xr8   r   )rJ   r\   r]  r   r  )rV  r   rQ  rW  s       r5   meta__pdist_backwardr\  ^  sW     
LLV 
LLX D0N0NOOr8   )rL  rK  c                  ^ ^^^	^
^ TR                  S5      nTR                  S5      nTR                  S5      nT R                  XVU45      m [        R                  " TR	                  5       S:H  S 5        [        R                  " TR	                  5       S:H  S 5        [        R                  " T R
                  TR
                  s=:H  =(       a    TR
                  :H  Os  UUU 4S j5        TR                  nTR                  m	US   m
US   m[        R                  " T	S   T
:H  =(       a    T	S   T:H  U	U
U4S j5        T R                  T R                  5       5      $ )	Nr   r   r   r-   c                      gNzbatch1 must be a 3D tensorr=   r=   r8   r5   rY   meta_baddbmm.<locals>.<lambda>q      ,Hr8   c                      gNzbatch2 must be a 3D tensorr=   r=   r8   r5   rY   r`  r  ra  r8   c                  V   > STR                    ST R                    STR                    3$ )Nz+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: r   )batch1batch2r   s   r5   rY   r`  u  s.    =djj\TZT`T`Saaklrlxlxkyzr8   c            	      .   > ST ST ST S    ST S    S3	$ Nz@Expected size for first two dimensions of batch2 tensor to be: [ro   z] but got: [r   r   ].r=   batch2_sizesbscontraction_sizes   r5   rY   r`  }  s5    t2&'|LO3DB|TUFWWY[r8   )r   r  rJ   r\   r|   rR   r   r   )r   re  rf  rL  rK  dim1dim2dim3batch1_sizesrk  rl  rm  s   ```      @@@r5   meta_baddbmmrr  j  s    ;;q>D;;q>D;;q>D;;D)*D	LL"$HI	LL"$HI	LL

fll22fll2z <<L<<L	aB#A	LLQ2E,q/5E"E	
 >>$))+&&r8   c                H    [         R                  " U [         R                  S9$ r   rh  r   r  s     r5   meta_bernoulliru    s     D0G0GHHr8   c                     U $ r2   r=   r   rQ  r  s      r5   meta_bernoulli_rx        Kr8   c                 H    [         R                  " U [         R                  S9$ r   rh  rw  s      r5   meta_bernoulli_pr{    s     D0G0GHHr8   c                 .    [         R                  " U 5      $ r2   rJ   r   rt  s     r5   meta_poissonr~         D!!r8   c                     [         R                  " XR                  5       :  S 5        [         R                  " U [         R                  S9n[         R                  " U 5      U4$ )Nc                      g)NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r=   r=   r8   r5   rY   6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>      \r8   r   )rJ   r\   r|   r   bool)r   observer_onfake_quant_onrunning_minrunning_maxscale
zero_pointaveraging_const	quant_min	quant_maxch_axisper_row_fake_quantsymmetric_quantmasks                 r5   $meta__fused_moving_avg_obs_fq_helperr    sM      
LL((*\ D

3DT"D))r8   c                 P  ^^^^ [         R                  " U R                  5       S:H  S 5        [         R                  " UR                  5       S:H  S 5        U R                  u  mmUR                  u  mm[         R                  " TT:H  UUUU4S j5        U R	                  TT5      $ )Nr   c                      g)Nza must be 2Dr=   r=   r8   r5   rY   meta_mm.<locals>.<lambda>      ~r8   c                      g)Nzb must be 2Dr=   r=   r8   r5   rY   r    r  r8   c            	      "   > ST ST  ST ST S3	$ )Nz/a and b must have same reduction dim, but got [ro   z] X [ri  r=   )M1M2NrI  s   r5   rY   r    s&    A!Brd%PRtSUVWUXXZ[r8   )rJ   r\   r|   r   r   )abr  r  r  rI  s     @@@@r5   meta_mmr    sz     
LLA56	LLA56GGEArGGEB	LL
b[ ;;q!r8   c                    ^ ^ U(       a)  [        UU 4S j[        T R                  5       5       5      $ [        R                  " T R
                  T5      $ )Nc              3   P   >#    U  H  oT;  a  TR                   U   OS v   M     g7f)r   Nr?  )rh   r   dimsr   s     r5   rj   +_compute_reduction_shape.<locals>.<genexpr>  s$     UDTqtmTZZ]:DTs   #&)r[   r   r   rB   compute_reduction_output_shaper   )r   r  r  s   `` r5   r  r    s7    UE$))DTUUU//

DAAr8   c                 :   [        U [        R                  R                  5      (       a  U R                  R
                  $ [        U S5      (       aK  [        U R                  S5      (       a0  U R                  R
                  S:w  a  U R                  R
                  $ g)Nry   rq   rw   r   )rf   rJ   _subclasses
FakeTensorfake_devicerq   hasattrry   )r/  s    r5   r   r     sp    &%++6677!!&&&!!FMM6**MM&(}}!!!r8   input_tensorr   r  dilationis_transposedgroupsoutput_paddingc                 P  ^^ S[         S[         S[         S[         S[         S[         4S jnS[         S[         S[         S[         S[         S[         S[         4S	 jn	UR                  S
S  n
U R                  S
S  mU(       a  XaR                  S   -  nO=UR                  S   nUR                  S   U-  U R                  S   :w  a  [        S5      eU R                  S   U/m[        U[        5      (       a  U/[        T5      -  nO![        U5      S:X  a  US   /[        T5      -  n[        U[        5      (       a  U/[        T5      -  nO![        U5      S:X  a  US   /[        T5      -  n[        U[        5      (       a  U/[        T5      -  nO![        U5      S:X  a  US   /[        T5      -  nS nU(       aI  [        U[        5      (       a  U/[        T5      -  nO$[        U5      S:X  a  US   /[        T5      -  nOUn[        [        T5      5       H[  nU(       a+  TR                  U	" TU   X=   XM   X   X-   X   5      5        M5  TR                  U" TU   X=   XM   X   X-   5      5        M]     [        R                  " [        S TS
S   5       5      UU4S j5        T$ )NlnrQ  r   r`  r  r.   c                 4    U SU-  -   X#S-
  -  -
  S-
  U-  S-   $ )aE  
Formula to apply to calculate the length of some dimension of the output

See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

Args:
    ln: length of the dimension
    p: padding in that dim
    d: dilation in that dim
    k: kernel size in that dim
    s: stride in that dim
Returns:
    The output length
r   r   r=   )r  rQ  r   r`  r  s        r5   _formula+calc_conv_nd_return_shape.<locals>._formula  s,     QU
Qa%[(1,2Q66r8   r3   c                 :    U S-
  U-  SU-  -
  X#S-
  -  -   U-   S-   $ )a  
Formula to apply to calculate the length of some dimension of the output
if transposed convolution is used.
See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

Args:
    ln: length of the dimension
    p: padding in that dim
    d: dilation in that dim
    k: kernel size in that dim
    s: stride in that dim
    op: output padding in that dim

Returns:
    The output length
r   r   r=   )r  rQ  r   r`  r  r3   s         r5   _formula_transposed6calc_conv_nd_return_shape.<locals>._formula_transposed  s0    " Q!|a!e#aq5k1B6::r8   r   r   r   zInvalid channel dimensionsc              3   *   #    U  H	  oS :  v   M     g7fr   Nr=   r  s     r5   rj   ,calc_conv_nd_return_shape.<locals>.<genexpr>=	  s     )=aE=   c                  .   > S[        T 5       STSS   S3$ )NzGiven input size per channel: z&. Calculated output size per channel: r   z. Output size is too small)r   )r  	ret_shapes   r5   rY   +calc_conv_nd_return_shape.<locals>.<lambda>>	  s(    0d =//8}o >#$r8   )r   r   r$  rf   r   r   r   r   rJ   r\   r}   )r  r8  r   r  r  r  r  r  r  r  kernel_sizeout_channelsoutput_padding_listr   r  r  s                 @@r5   calc_conv_nd_return_shaper    s   7S 7S 7S 7S 7S 7S 7"; ; ; ; ; ; ;QT ;& ,,qr"Kab!DQ/||A<<?V#|'9'9!'<<;<<##A&5I&'""CI%	V	)s4y('7##)c$i'	W	1:,T*(G$$:D	)	X!	QK=3t9,/3ng..#1"2SY"> A%#1!#4"5D	"A"03t9#GJKNI'*	 a'*hk;>69U " 
LL)9QR=))	$ r8   c                 b    [         R                  R                  U 5      [         R                  :H  $ r2   rJ   _prims_commonsuggest_memory_formatchannels_lasttens    r5   is_channels_lastr  F	  s$    44S9U=P=PPPr8   running_meanrunning_vartrainingexponential_average_factorepsilonc                 z  ^  T R                   nUb  UR                   OUR                   n	Ub  UR                   OUR                   n
U 4S jnT R                  U5      R                  U" 5       S9nU(       a#  T R                  U	5      nT R                  U
5      nO"T R                  S5      nT R                  S5      nXU4$ )Nc                     > [        T 5      (       a  [        R                  $ T R                  [        R                  S9(       a  [        R                  $ [        R                  $ r   )r  rJ   r  r]  r   )r  s   r5   pick_memory_format2meta_miopen_batch_norm.<locals>.pick_memory_format]	  sI    L))&&&%%E4K4K%L***&&&r8   r   r   )r   r   r%  )r  r8  r:  r  r  r  r  r  r   save_mean_shapesave_var_shaper  r   	save_meansave_vars   `              r5   meta_miopen_batch_normr  J	  s     ""I -9,Dl((&,,O*5*A[&&v||N' 
 
 
+
.
.=O=Q
.
RC **?;	)).9 **40	))$/8##r8   c	           
         ^ ^ U U4S jn	[        T TUUUUUU(       a  UOS 5      n
SnSnT R                  U5      S:X  a  SX'   T R                  U
5      nUR                  U	" 5       S9nU$ )Nc                    > [        T 5      S:X  a1  [        T 5      (       d  [        T5      (       a  [        R                  $ O [        T 5      (       a  [        R                  $ T R	                  [        R
                  S9(       a  [        R
                  $ T R	                  [        R                  S9(       a  [        R                  $ g Nr   r   )r   r  rJ   r  r]  r   preserve_format)r  r8  s   r5   r  %meta_conv.<locals>.pick_memory_format|	  s    |$.--1A&1I1I*** 2J  --***%%E4K4K%L***''e6K6K'L((( Mr8   r   r   r   )r  r   r   r%  )r  r8  r:  r   r  r  r  r  r  r  	shape_outinput_channels_dimoutput_channels_dimr   s   ``            r5   	meta_convr  p	  s    
) *'T	I +,1)*	&

 
 
+C
&&13&
4CJr8   mkldnnc
           
          [        XXCUSU/ 5      n
U R                  U
5      n[        R                  nU R	                  5       S:X  a  [        R
                  nUR                  US9nU$ )NFr6  r   )r  r   rJ   r  r|   channels_last_3dr%  )r  r8  r:  r  r   r  r  attrscalars	algorithmr  r   out_memory_formats                r5   meta_mkldnn_convolution_defaultr  	  sl     .&8UFB
	 $$Y/!//" % 6 6ff#4f5
r8   c                 b    U R                  / U R                  S S QUR                  S   P75      $ Nr   r   r   r   )r  r8  r:  r  r  r  s         r5   meta_linear_pointwise_defaultr  	  s5     %%&Q(:(:3B(?&Qa&QRRr8   mklc                 b    U R                  / U R                  S S QUR                  S   P75      $ r  r  )r  packed_weightorig_weightr:  r   s        r5   meta_mkl_linearr  	  s:    ))@,$$Sb)@;+<+<Q+?@ r8   onednnc           
         [        U UUUU	SU
S 5      nU[        R                  [        R                  [        R                  [        R
                  4;   d   eU R                  UUS9nUR                  [        R                  S9nU$ )NFr   r   )	r  rJ   rY  r[  uint8r?  r   r%  r  )rG   x_scalex_zpww_scalew_zpr:  r   r  r  r  output_scaleoutput_zero_pointoutput_dtyper  r  r  r  r   s                      r5   meta_qconv2d_pointwiser  	  s|    ( .	
	 u~~u{{EJJWWWWkk)<k8ff5#6#6f7
r8   c                     US:X  d   eU$ )Nsumr=   )rG   r  r  r  r  r  accumr:  r   r  r  r  r  r  r  accum_scaleaccum_zero_pointbinary_op_namerK  unary_op_nameunary_op_argsunary_op_algorithms                         r5   meta_qconv2d_pointwise_binaryr  	  s    2 &&&r8   c                     [        U R                  5      nUR                  S   US'   U	[        R                  [        R                  [        R
                  [        R                  4;   d   eU R                  XS9nU$ )Nr   r   r   )r   r   rJ   rY  r[  r?  r  r   )rG   r  r  r  r  r  r:  r  r  r  post_op_namepost_op_argspost_op_algorithmrb  r   s                  r5   meta_qlinear_pointwiser  
  s_    " AGG}771:Ru~~uzz5;;WWWWkk,k;
r8   c                    US:X  a  U$ [        U R                  5      nUR                  S   US'   U
[        R                  [        R                  [        R
                  [        R                  4;   d   eU R                  UU
S9nU$ )Nr  r   r   r   )r   r   rJ   rY  r[  r  r?  r   )rG   r  r  r  r  r  x_2r:  r  r  r  x2_scalex2_zpr  rK  r  r  r  rb  r   s                       r5   meta_qlinear_pointwise_binaryr  %
  sn    , U"JAGG}771:Ru~~u{{EJJWWWWkk,lk;
r8   c                 v    [        U R                  5      nUR                  S   US'   U R                  U5      nU$ )Nr   r   )r   r   r   )rG   r  r:  rb  r   s        r5   meta_linear_dynamic_fp16r  D
  s6     AGG}771:Rkk,'
r8   	quantizedr   c                 "   [        XX#XE5      u  nnnU R                  5       S:X  a  U R                  S5      OSn	[        R                  n
U R                  5       S:X  a  XgU/nOXXx/n[        R
                  " UU R                  U R                  U
S9$ Nr  r   r-   ry  )#max_pool2d_checks_and_compute_shaper|   r   rJ   r  r   rR   ry   r   r  r   r  r  	ceil_modenInputPlaneoutputHeightoutputWidthr  r   r   s               r5   meta_quantized_max_pool2dr  U
  s     0
		
 $)99;!#3B++99;!{;DCD{{++<<'	
 	
r8   c                    [         R                  " U R                  5       S:H  SU R                  5        S35        [         R                  " UR                  5       S:H  SUR                  5        S35        [         R                  " U R                  [         R                  [         R
                  [         R                  4;   SU R                   35        [         R                  " UR                  [         R                  :H  SUR                   35        [         R                  " UR                  [         R                  :H  SUR                   35        [         R                  " UR                  U R                  :H  SUR                   35        U R                  U R                  S	5      UR                  S	5      U R                  S
9$ )Nr   zx must be a 2D tensor, got Dzw must be a 2D tensor, got #expected x to be f32/f16/bf16, got expected w to be uint8, got z q_group_size must be int64, got z5q_scale_and_zeros must have the same dtype as x, got r   r   )rJ   r\   r|   rR   rY  rZ  r[  r  r   r   r   rG   r  q_group_sizeq_scale_and_zeross       r5   meta_int4mm_packed_weight_cpur'  r
  s@   QUUW\%@	#KLQUUW\%@	#KLGGu}}ennEE1!'';	
 	QWW+/KAGG9-UV%++-.|/A/A.BC	
 	##qww.CDUD[D[C\]	
 {{166!9affQiqww{??r8   c                    ^ ^^^ [         R                  " T R                  5       T:H  =(       a    T R                  T   T:H  UUUU 4S j5        g )Nc                  j   > ST  ST ST S3STR                  5        ST STR                  T    3-   $ )NzExpected a tensor of dimension z and tensor.size[z] == ro   zbut got : dimension z] = r|   r   )r|   dim_sizer   r/  s   r5   rY    check_dim_size.<locals>.<lambda>
  sN    1#6GzQVW[V\\^_ .?zfll[cNdMe
fgr8   )rJ   r\   r|   r   )r/  r|   r+  r   s   ````r5   check_dim_sizer-  
  s6    	LL

>X 6$ >	gr8   c                   ^  S nU" SU5      u  p[         R                  " [        U5      S;   S 5        [         R                  " T R                  [         R                  [         R
                  [         R                  [         R                  4;  U 4S j5        [        U5      S:X  a  XpO$[        U5      S:X  a
  US   US   pOU" SU5      u  pU" S	U5      u  p[         R                  " US L =(       d    US:g  S
 5        T R                  5       S:X  a  T R                  S5      OSnT R                  S5      nT R                  S5      nT R                  S5      n[        UXU
SU5      n[        UXUSU5      n[        R                  " T 5      n[        T UU	U
UUUSSUUUUUU5        T R                  5       S:X  a  UUU/nOXUU/n[         R                  " UT R                  T R                  US9$ )Nc                    ^  [         R                  " [        U5      S;   U 4S j5        US   n[        U5      S:X  a  UOUS   nX#4$ )Nr   r   c                     > ST  S3$ )Nzavg_pool2d: 4 must either be a single int, or a tuple of two intsr=   r  s   r5   rY   1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>
      l4&(\]r8   r   r   rJ   r\   r   r  r  HWs   `   r5   unpackmeta_avg_pool2d.<locals>.unpack
  E    H]	
 FSQACFtr8   r  r   r   r   c                      gNzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr=   r=   r8   r5   rY   !meta_avg_pool2d.<locals>.<lambda>
      ar8   c                  @   > ST R                   R                  5        S3$ )Nz""avg_pool2d" not implemented for ''rR   __str__r   s   r5   rY   r@  
      6u{{7J7J7L6MQQr8   r   r   r   r  c                      gNzdivisor must be not zeror=   r=   r8   r5   rY   r@  
      *r8   r  r  r  r   r-   ry  )rJ   r\   r   rR   r  uint16uint32uint64r|   r   pooling_output_shaperB   r  pool2d_shape_checkr   ry   )r   r  r   r  r  count_include_paddivisor_overrider:  kHkWdHdWpadHpadWr  r  inputHeight
inputWidthr  r  r   r   s   `                     r5   meta_avg_pool2dr[  
  s    M;/FB	LLFy a 
LLEKKu||U\\RRQ 6{aB	V	F1IB&)	7+JD	LLD 9$4$9*
  %yy{a/UZZ^QF**R.K**R.KBJ'Rr1iPL&z2RINK//6M



		$ yy{a\;7\;?;;kk||#	 r8   c                     [        U UUUUUUSSU	U
UUUU5        U R                  5       nU	n[        XUS-
  U5        [        XUS-
  U5        [        XUS-
  U5        g )Nr   r-   r   )rP  r|   r-  )r   
gradOutputr  rS  rT  rU  rV  rW  rX  r  rY  rZ  r  r  
mem_formatr   nOutputPlanes                    r5   avg_pool2d_backward_shape_checkr`  
  s{    " 



		$ 99;DL:TAX|<:TAX|<:TAX{;r8   c                    [         R                  " [        U5      S:H  =(       d    [        U5      S:H  S 5        US   n[        U5      S:X  a  UOUS   n	[         R                  " [        U5      S:H  =(       d#    [        U5      S:H  =(       d    [        U5      S:H  S 5        [        U5      S:X  a  UOUS   n
[        U5      S:X  a  U	O[        U5      S:X  a  U
OUS   n[         R                  " [        U5      S:H  =(       d    [        U5      S:H  S 5        US   n[        U5      S:X  a  UOUS   n[         R                  " US L =(       d    US:g  S 5        UR                  nUR	                  5       S:X  a  US	   OSnUS
   nUS   nUS   n[        UXU
SU5      n[        UXUSU5      n[        R                  " U5      n[        UU UUU	U
UUUUUUUUU5        [         R                  " UUR                  UR                  US9$ )Nr   r   c                      g)NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr=   r=   r8   r5   rY   *meta_avg_pool2d_backward.<locals>.<lambda>  s    ]r8   r   c                      gr?  r=   r=   r8   r5   rY   rc  !  rA  r8   c                      g)NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr=   r=   r8   r5   rY   rc  '  s    Yr8   c                      grI  r=   r=   r8   r5   rY   rc  .  rJ  r8   r  r  rK  r  r   ry  )rJ   r\   r   r   r|   rO  rB   r  r`  r   rR   ry   )gradOutput_r   r  r   r  r  rQ  rR  rS  rT  rU  rV  rW  rX  
input_sizer  r  rY  rZ  r  r  r^  s                         r5   meta_avg_pool2d_backwardri    s    
LLKA6[!1Q!6] 
QB;1$+a.B	LLFq@CK1,@Fq0@a 6{aVAYB6{a3v;!+;RB	LLG.S\Q.Y 1:Dw<1$4'!*D	LLD 9$4$9*
 J$yy{a/Z^QFR.KR.KBJ'Rr1iPL&z2RINK,,U3J#



$ ;;kk|| 	 r8   c                 n  ^  [         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n	[         R                  " U(       + =(       d    [        U5      S;   S 5        [         R                  " T R                  [         R                  [         R
                  [         R                  [         R                  4;  U 4S j5        U(       d  UOUS   n
U(       d  UO[        U5      S:X  a  U
OUS   nU(       d  U	O[        U5      S:X  a  U
OUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " T R                  S	;   S
 5        [         R                  " U(       + =(       d    US:g  S 5        T R                  S5      nT R                  S5      nT R                  S5      nT R                  S5      nT R                  S5      n[        UX}U
SU5      n[        UXUSU5      n[        UXUSU5      n[        T UUUU	U
UUUUUSSSUUUUUUSSS9  T R                  S:X  a  T R                  UUUU45      $ T R                  UUUUU45      $ )Nr   r-   c                      gNzFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr=   r=   r8   r5   rY   !meta_avg_pool3d.<locals>.<lambda>c      Xr8   r   r   r   c                      gNzJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr=   r=   r8   r5   rY   rn  k  r  r8   c                  @   > ST R                   R                  5        S3$ )Nz""avg_pool3d" not implemented for 'rC  rD  rF  s   r5   rY   rn  o  rG  r8   c                      gNzBavg_pool3d: padding must be a single int, or a tuple of three intsr=   r=   r8   r5   rY   rn  w      Tr8   r  r6  c                      gNz9non-empty 4D or 5D (batch mode) tensor expected for inputr=   r=   r8   r5   rY   rn        Kr8   c                      grI  r=   r=   r8   r5   rY   rn    rJ  r8   r  rK  r  r   zavg_pool3d()T)check_input_sizer  )rJ   r\   r   rR   r  rL  rM  rN  r   r   rO  pool3d_shape_checkr   )r   r  r   r  r  rQ  rR  kTrS  rT  dTrU  rV  padTrW  rX  r  nslicesitimeiheightiwidthotimeoheightowidths   `                       r5   meta_avg_pool3dr  V  s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
+c&kV+\ 
LLEKKu||U\\RRQ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LL5 0A 5*
 ZZ]FjjnGJJrNEjjnGZZ^F "aCE"7Bb!YGG!&"B9EF





			-2 zzQ@AAHIIr8   c                 B   [         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n	[        U5      S:X  a  UOUS   n
[         R                  " U(       + =(       d    [        U5      S;   S 5        U(       d  UOUS   nU(       d  U	O[        U5      S:X  a  UOUS   nU(       d  U
O[        U5      S:X  a  UOUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " UR                  S;   S	 5        [         R                  " U(       + =(       d    US:g  S
 5        UR	                  S5      nUR	                  S5      nUR	                  S5      nUR	                  S5      n[        UXUSU5      n[        UXUSU5      n[        UU
UUSU5      n[        UU UUU	U
UUUUUUUUUUUUS5        UR                  UR                  5      $ )Nrk  c                      grm  r=   r=   r8   r5   rY   *meta_avg_pool3d_backward.<locals>.<lambda>  ro  r8   r   r   r   c                      grq  r=   r=   r8   r5   rY   r    r  r8   c                      grt  r=   r=   r8   r5   rY   r    ru  r8   rv  c                      grx  r=   r=   r8   r5   rY   r    ry  r8   c                      grI  r=   r=   r8   r5   rY   r    rJ  r8   r  rK  r  r   zavg_pool3d_backward())	rJ   r\   r   r   r   rO  avg_pool3d_backward_shape_checkr   r   )r  r   r  r   r  r  rQ  rR  r}  rS  rT  r~  rU  rV  r  rW  rX  r  r  r  r  otime_for_shape_checkoheight_for_shape_checkowidth_for_shape_checks                           r5   meta_avg_pool3d_backwardr    s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
+c&kV+\ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LL5 0A 5*
 jjnGJJrNEjjnGZZ^F0"aS27Bb!YW1&"dB9U#





', ??5;;''r8   c                 8  ^  [         R                  " T R                  S:H  =(       d    T R                  S:H  U 4S j5        T R                  S S [	        U5      -   n[
        R                  " T 5      n[         R                  " UT R                  T R                  US9$ )Nr-   r  c                  "   > ST R                    3$ )Nz"Expected 3D or 4D tensor, but got r?  r   s   r5   rY   *meta_adaptive_avg_pool2d.<locals>.<lambda>      4TZZLAr8   r  ry  )
rJ   r\   r   r   r[   rB   r  r   rR   ry   )r   output_sizerb  r   s   `   r5   meta_adaptive_avg_pool2dr     s|    	LL		Q($))q.A ::cr?U;%77L//5M ;;jj{{#	 r8   c                    ^  [         R                  " T R                  S:H  =(       d    T R                  S:H  U 4S j5        T R                  T R                  S S [        U5      -   5      $ )Nr  r6  c                  "   > ST R                    3$ )Nz"Expected 4D or 5D tensor, but got r?  r   s   r5   rY   *meta_adaptive_avg_pool3d.<locals>.<lambda>  r  r8   rK  )rJ   r\   r   r   r   r[   )r   r  s   ` r5   meta_adaptive_avg_pool3dr    sO    	LL		Q($))q.A >>$**Sb/E+,>>??r8   c                   ^ ^^ T R                   n[        SU5       H1  m[        R                  " T R	                  T5      S:  U U4S j5        M3     [        R                  " US:H  =(       d    US:H  U4S j5        [        R                  " TR
                  T R
                  :H  U U4S j5        [        R                  n[        T5      (       a  [        R                  nTR                  TR                  5      R                  US9$ )	Nr   r   c                  *   > ST R                    ST S3$ )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions,  with dimension  being emptyr?  )grad_outr   s   r5   rY   4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>!  s$     66>nn5EEUVWUXXdfr8   r-   r  c                  "   > ST R                    3$ )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got r?  r   s   r5   rY   r  &  s    TUYU_U_T`ar8   c                  <   > STR                    ST R                    3$ Nexpected dtype z! for `grad_output` but got dtype r   )r  r   s   r5   rY   r  *  s    /$**-Nx~~N^_r8   r   )r   r   rJ   r\   r   rR   r   r  r  r   r   r%  )r  r   r   r   r   s   ``  @r5   "meta__adaptive_avg_pool2d_backwardr    s    ==D1d^MM!q f	
  
LL	TQYa 
LL

hnn$_ ++M++>>$**%((}(EEr8   c                 `    [        U S5        [        R                  " U[        R                  S9$ )Nadaptive_avg_pool3d_backwardr   )!_adaptive_pool_empty_output_checkrJ   r   r  r  r   s     r5   "meta__adaptive_avg_pool3d_backwardr  2  s(     &k3QRD0N0NOOr8   r  c                    ^ ^^ T R                   n[        SU5       H2  m[        R                  " T R	                  T5      S:  UU U4S j5        M4     g )Nr   r   c                  .   > T  STR                    ST S3$ )Nzc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes r  r  r?  )r  r  r   s   r5   rY   3_adaptive_pool_empty_output_check.<locals>.<lambda>>  s*    * --8->->,??OPQsR^`r8   )r   r   rJ   r\   r   )r  r  r   r   s   `` @r5   r  r  9  sB    D1d^Q!#	
 r8   c                   ^ ^ T R                   n[        R                  " US;   U 4S j5        [        SU5       H1  m[        R                  " T R	                  T5      S:  UU 4S j5        M3     [        R                  " [        U5      S:H  S 5        SnSnSnT R                   S:X  a  T R	                  S5      nUS-  nT R	                  US-
  5      nUu  pgT R                   S	:X  a6  XVU4nT R                  U5      n	T R                  U[        R                  S
9n
X4$ XEXg4n[        R                  " T 5      nT R                  U5      R                  US9n	T R                  U[        R                  S
9R                  US9n
X4$ )Nr-   r  c                  "   > ST R                    3$ )Nz:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: r?  rF  s   r5   rY   *meta_adaptive_max_pool2d.<locals>.<lambda>K      LU[[MZr8   r   r   c                  *   > STR                    ST  S3$ )Nzjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  r?  r   r   s   r5   rY   r  P       '',{{m3CA3lTr8   r   c                      g)NzCadaptive_max_pool2d(): internal error: output_size.size() must be 2r=   r=   r8   r5   rY   r  X      Ur8   r  r-   r   r   )r   rJ   r\   r   r   r   r   r   rB   r  r%  )r   r  r   dimHsizeBsizeDosizeHosizeWr   r   r   r   r   s   `           @r5   meta_adaptive_max_pool2dr  E  sk    ::D	LLZ 1d^JJqMA	
  
LLKAU
 DEEzzQ

1	JJtax E NFzzQF+	ooi(//)5;;/?|62	33E:ooi(++-+H//)5;;/?BB' C 
 |r8   c                 T  ^ ^ T R                   n[        R                  " US;   U 4S j5        [        T S5        [        R                  " TR                  T R                  :H  U U4S j5        [
        R                  " T5      nTR                  TR                  5      R                  US9$ )Nr  c                  "   > ST R                    3$ )NzKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: r?  r  s   r5   rY   3meta_adaptive_max_pool2d_backward.<locals>.<lambda>{  s    ]^i^o^o]pqr8   adaptive_max_pool2d_backwardc                  <   > STR                    ST R                    3$ r  r   )r  r   s   r5   rY   r    s    /%++.OP[PaPaObcr8   r   )
r   rJ   r\   r  rR   rB   r  r   r   r%  )r  r   r   r   r   s   ``   r5   !meta_adaptive_max_pool2d_backwardr  u  s     D	LLq
 &k3QR	LL{(((c
 //6M??5;;'***GGr8   c                   ^ ^ T R                   n[        R                  " US;   U 4S j5        [        SU5       H1  m[        R                  " T R	                  T5      S:  UU 4S j5        M3     [        R                  " [        U5      S:H  S 5        SnSnSnUS:X  a  T R	                  S5      nUS-  nT R	                  U5      nUu  pgnUS	:X  a  XVXx4n	OXEXgU4n	T R                  U	5      n
T R                  U	[        R                  S
9nX4$ )Nrv  c                  "   > ST R                    3$ )Nz:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: r?  rF  s   r5   rY   *meta_adaptive_max_pool3d.<locals>.<lambda>  r  r8   r   r   c                  *   > STR                    ST  S3$ )Nzjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  r?  r  s   r5   rY   r    r  r8   r-   c                      g)NzCadaptive_max_pool3d(): internal error: output_size.size() must be 3r=   r=   r8   r5   rY   r    r  r8   r6  r  r   )r   rJ   r\   r   r   r   r   r   )r   r  r   dimDr  r  osizeTr  r  r   r   r   r   s   `           @r5   meta_adaptive_max_pool3dr    s    ::D	LLZ 1d^JJqMA	
  
LLKAU
 DEEqy

1	JJtE(FFqyF3	66:	
//)
$Cooiu{{o;G<r8   c                 P    [        U S5        UR                  UR                  5      $ )Nadaptive_max_pool3d_backward)r  r   r   )r  r   r   s      r5   !meta_adaptive_max_pool3d_backwardr    s"     &k3QR??5;;''r8   c                 @    Uc  [        S5      eU R                  U5      $ )Nz:cannot repeat_interleave a meta tensor without output_size)r$  r   )repeatsr  s     r5   meta_repeat_interleave_Tensorr    s%    WXX[))r8   c                     U R                   R                  (       d   eUR                   R                  (       d   e[        U R                  UR                  5      nU R	                  U[        U R                   5      S9$ r  )rR   r   r$   r   r   r   )realimagr   s      r5   meta_complexr    s[     ::''''::''''!$**djj9I>>)+Ftzz+R>SSr8   )
fill_valuer  c                \    U R                  XR                  5       4[        R                  S9$ r  )r   r|   rJ   r   )r   r   r  s      r5   nonzero_staticr    s$     >>4,EJJ>??r8   c                    [         R                  " [        R                  S 5        [         R                  " U R                  5       U R                  5       4SU R                  5       4[         R                  U R                  S9$ )Nc                      g)NaY  The register_meta function for torch.nonzero() raises unimplemented by default, as a correct data-independent implementation does not exist. This implementation returns a fake value, assuming all elements of the tensor are non-zero. To enable this registration, please set 'torch.fx.experimental._config.meta_nonzero_assume_all_nonzero' to True.r=   r=   r8   r5   rY   nonzero.<locals>.<lambda>  s     Sr8   r   rR   ry   )	rJ   _check_not_implemented
exp_configmeta_nonzero_assume_all_nonzeror-  r   r|   r   ry   r   s    r5   nonzeror    sf     
  22	S 	txxz"	
DJJLjj{{	 r8   c           
      j	  ^ ^^^^^^^^ [         R                  " [        T5      S 5        / n[        T5       GH  u  mmTGbr  [         R                  " TR                  [         R
                  [         R                  [         R                  [         R                  4;   S 5        TR                  [         R                  [         R                  4;   a  TR                  5       n[        U5      m[         R                  " TTR                  -   T R                  :*  U 4S j5        [        TR                  5       Hc  m[         R                  " TR                  T   T R                  TT-      :H  UUUUU 4S j5        UR                  UR                  ST5      5        Me     GMi  UR                  T5        GM}  UR                  T5        GM     Um[         R                  " [        T5      T R                  :*  UU 4S j5        SS KJn  [%        UR&                  " T6 5      m[        T5      T R                  :  a,  TR                  S 5        [        T5      T R                  :  a  M,  SnSnT H&  mUS:X  a	  Tb  SnM  M  US:X  a	  Tc  S	nM  M!  Tc  M&    O   S
nU(       d  / n/ n[        T5       H-  u  mmTc  M  UR                  T5        UR                  T5        M/     [        T5       H-  u  mmTb  M  UR                  T5        UR                  T5        M/     T R)                  U5      m Um/ m/ m/ m[        T5       He  u  n	mTcG  T(       a   TR                  T R                  U	   5        M0  TR                  T R                  U	   5        MP  [%        TR                  5      mMg     UUU4S jn
T R+                  TT-   T-   5      nSSKJn  U" T R1                  5       S:H  5      (       a  U$ U
" T 5      n[2        R4                  " U5      n[%        U5      [%        [        [        U5      5      5      :w  a  [2        R6                  " UR                  U5      n[2        R8                  " U5      n[2        R6                  " U[2        R:                  " U5      5      nUR=                  UR?                  5       U5      nU$ )Nc                      g)Nz#at least one index must be providedr=   r=   r8   r5   rY   #meta_index_Tensor.<locals>.<lambda>  s    (Mr8   c                      g)Nz?tensors used as indices must be long, int, byte or bool tensorsr=   r=   r8   r5   rY   r    s    Yr8   c                  "   > ST R                    3$ )N)too many indices for tensor of dimension r	  r   s   r5   rY   r    s    G		{Sr8   c            	      N   > STR                    ST  STR                    STT-    3$ )NzThe shape of the mask z
 at index z0 does not match the shape of the indexed tensor r?  )r   r   jr`  r   s   r5   rY   r    s:    "8ZPQs SJJN**U_`ade`e_f!hr8   r   c                  <   > STR                    S[        T 5       S3$ )Nr  z (got rp   )r   r   )r   r   s   r5   rY   r    s    ;DII;fSQX\NZ[\r8   r   Fr   Tc                    > TT-   T-   n[        U R                  5       5      nS/[        T5      -  U[        T5      [        U R                  5      [        T5      -
  & U R	                  X5      $ )z9
This follows restride_src in TensorAdvancedIndexing.cpp
r   )r   r   r   r   rA  )r   r   r   after_shapebefore_shapereplacement_shapes      r5   _restride_src(meta_index_Tensor.<locals>._restride_srcG  sm     00;>t{{}%KL#PSQ
 K
L!C

Oc+6F$FG u..r8   guard_size_oblivious) rJ   r\   r  	enumeraterR   r   r   r?  r  r   r   r   r   r   r   selecttorch._refs_refsr   r%   r   r   r"  r  r   rB   3compute_elementwise_output_logical_to_physical_perm
apply_permr   invert_permrA  r   )r   r   r  r  refsstatehas_contiguous_subspacer  transposed_indicesr|   r  r   r  restrided_selfperm
perm_shaper2  r  r  r   r   r  r`  r  s   ``               @@@@@@@r5   meta_index_Tensorr    s   	LLg MN &(Fg&5LL

EIIuzz5::NNY {{uzz5::66--/K""

Ndii/S uzz*A&&A$**QU*;;h h
 MM'..A"67 + e$MM% / '0 G	LLG		!\
 4(('23G
g,
"t g,
" E#A:  !aZ}     #'
 #!'*HAu A"))%0 + "'*HAu}A"))%0 + ||D!$ !LK#%(
U= ""4::c?3##DJJsO4 $U[[ 1 )	/ ..(99KG
HCJDJJLA-..

 #4(NDD^TD DzT%D	*++%%cii6
66zB
%%j%2C2CD2IJ
nnSXXZ4Jr8   c                     S nS nS nU
S   (       a  U R                  UR                  5       5      nU
S   (       a  U R                  UR                  5       5      nU
S   (       a  U R                  U5      nXU4$ )Nr   r   r   r   r   )grad_output_input_weight_bias_sizes_optr   r  r  
transposedr  r  output_maskbackend_grad_inputbackend_grad_weightbackend_grad_biass                 r5   meta_convolution_backwardr  h  sv      1~)33FKKMB1~*44W\\^D1~(22>B5FGGr8   c                  ^^ TR                  S5      nTR                  S5      nU R                  XV45      n [        R                  " TR	                  5       S:H  S 5        [        R                  " TR	                  5       S:H  S 5        [        R                  " TR                  S5      TR                  S5      :H  UU4S j5        [        R                  " TR                  S5      TR                  S5      :H  UU4S j5        [        R                  " U R                  S5      U:H  =(       a    U R                  S5      U:H  S	 5        U R                  U R                  5       5      $ )
Nr   r   r-   c                      gr_  r=   r=   r8   r5   rY   meta_addbmm.<locals>.<lambda>  ra  r8   c                      grc  r=   r=   r8   r5   rY   r    ra  r8   r   c                  P   > ST R                  S5       STR                  S5       3$ )Nz8batch1 and batch2 must have same number of batches, got r   r   r   re  rf  s   r5   rY   r    s,    J6;;WX>JZZ_`f`k`klm`n_opr8   c            
         > ST R                  S5       ST R                  S5       STR                  S5       STR                  S5       S3	$ )Nz#Incompatible matrix sizes for bmm (r   rG   r   r   rp   r   r  s   r5   rY   r    sL    1&++a.1A6;;q>BR S;;q>"!FKKN#316r8   c                      g)Nz.self tensor does not match matmul output shaper=   r=   r8   r5   rY   r    s    @r8   )r   r  rJ   r\   r|   r   )r   re  rf  rL  rK  rn  ro  s    ``    r5   meta_addbmmr    s     ;;q>D;;q>D;;|$D	LL"$HI	LL"$HI	LLA&++a.(p 
LLA&++a.(	
 
LL		!51!5@ >>$))+&&r8   )
grad_scale	found_infc       	         p   ^ XX#XE4 H,  m[         R                  " [        T[        5      U4S j5        M.     g )Nc                      > S[        T 5       3$ Nz'exponent must be a tensor list but got rq   ls   r5   rY   #meta__fused_adam_.<locals>.<lambda>      =d1gYGr8   rJ   r\   rf   r   )r   gradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepslrbeta1beta2weight_decayepsamsgradmaximizer  r  r!  s                  @r5   meta__fused_adam_r1    s0    & 8/Oq$G	
 Pr8   c       	            ^ XX#XE4 H,  m[         R                  " [        T[        5      U4S j5        M.     S nU" U 5      U" U5      U" U5      U" U5      U" U5      4$ )Nc                      > S[        T 5       3$ r  r  r   s   r5   rY   "meta__fused_adam.<locals>.<lambda>  r#  r8   c                 Z    U  Vs/ s H  n[         R                  " U5      PM     sn$ s  snf r2   r}  )tensor_listr  s     r5   empty_like_list)meta__fused_adam.<locals>.empty_like_list  s%    -89[  #[999s    (r$  )r   r%  r&  r'  r(  r)  r*  r+  r,  r-  r.  r/  r0  r  r  r7  r!  s                   @r5   meta__fused_adamr9    si    & 8/Oq$G	
 P: 	!$( r8   c                 t  ^ ^ [         R                  " T R                  5       S:H  S 5        [         R                  " TR                  5       S:H  S 5        [         R                  " T R                  [         R                  L U 4S j5        [         R                  " TR                  [         R                  L U4S j5        [         R                  " T R                  S5      TR                  S5      :H  U U4S j5        T R                  T R                  S5      TR                  S5      4[         R                  S	9$ )
Nr   c                      g)Nza must be a 2D tensorr=   r=   r8   r5   rY   meta__int_mm.<locals>.<lambda>      '>r8   c                      g)Nzb must be a 2D tensorr=   r=   r8   r5   rY   r<    r=  r8   c                  "   > ST R                    3$ )Nzexpected self to be int8, got r   )r  s   r5   rY   r<        0	:r8   c                  "   > ST R                    3$ )Nzexpected mat2 to be int8, got r   )r  s   r5   rY   r<    r@  r8   r   r   c            
         > ST R                  S5       ST R                  S5       STR                  S5       STR                  S5       S3	$ )Nz'Incompatible matrix sizes for _int_mm (r   rG   r   r   rp   r   r  r  s   r5   rY   r<    sH    5affQi[!&&) M66!9+Qqvvayk,r8   r   )rJ   r\   r|   rR   r?  r   r   r@  rC  s   ``r5   meta__int_mmrD    s     
LLA>?	LLA>?	LL	5::: 
LL	5::: 
LL	q	QVVAY	
 ;;q	166!9-U[[;AAr8   c                 d  ^  [         R                  " T R                  5       S:H  S 5        [         R                  " T R                  [         R                  L U 4S j5        T R                  S5      nT R                  S5      S-  nT R                  US-  X1S-  -  SUS-  4[         R                  S	9$ )
Nr   c                      gNzw must be a 2D tensorr=   r=   r8   r5   rY   2meta__convert_weight_to_int4pack.<locals>.<lambda>  r=  r8   c                  "   > ST R                    3$ Nr#  r   r  s   r5   rY   rH        .qwwi8r8   r   r      rX      r   )rJ   r\   r|   rR   r  r   r   r@  r  inner_k_tilesr  r`  s   `   r5    meta__convert_weight_to_int4packrQ    s    	LLA>?	LL	5;;8 	
q	A	q	AA;;F"$%Q		
 kk   r8   c                 H  ^  [         R                  " T R                  5       S:H  S 5        [         R                  " T R                  [         R                  L U 4S j5        T R                  S5      nT R                  S5      nT R                  X#S-  4[         R                  S9$ )Nr   c                      grG  r=   r=   r8   r5   rY   :meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>  r=  r8   c                  "   > ST R                    3$ Nzexpected w to be int32, got r   rK  s   r5   rY   rT    rL  r8   r   r   r   )rJ   r\   r|   rR   r@  r   r   r  rO  s   `   r5   (meta__convert_weight_to_int4pack_for_cpurW  
  s}    	LLA>?	LL	5;;8 	
q	A	q	A;;	
Fkk   r8   c                 6  ^ ^ [         R                  " T R                  5       S:H  S 5        [         R                  " TR                  5       S:H  S 5        [         R                  " T R                  [         R                  [         R
                  [         R                  4;   U 4S j5        [         R                  " TR                  [         R                  L U4S j5        T R                  T R                  S5      TR                  S5      S-  T R                  S	9$ )
Nr   c                      gNzx must be a 2D tensorr=   r=   r8   r5   rY   *meta__weight_int4pack_mm.<locals>.<lambda>  r=  r8   r  c                      g)Nzw must be a 4D tensorr=   r=   r8   r5   rY   r[    r=  r8   c                  "   > ST R                    3$ Nr"  r   rG   s   r5   rY   r[        5aggY?r8   c                  "   > ST R                    3$ rV  r   rK  s   r5   rY   r[  #  rL  r8   r   rM  r   )
rJ   r\   r|   rR   rY  rZ  r[  r@  r   r   r$  s   ``  r5   meta__weight_int4pack_mmrb    s    	LLA>?	LLA>?	LL	EMM5==%..AA? 
LL	5;;8 ;;qvvay!&&)a-qww;??r8   c                 0  ^ ^ [         R                  " T R                  5       S:H  S 5        [         R                  " TR                  5       S:H  S 5        [         R                  " T R                  [         R                  [         R
                  [         R                  4;   U 4S j5        [         R                  " TR                  [         R                  L U4S j5        T R                  T R                  S5      TR                  S5      T R                  S9$ )Nr   c                      grZ  r=   r=   r8   r5   rY   2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>*  r=  r8   c                      grG  r=   r=   r8   r5   rY   re  +  r=  r8   c                  "   > ST R                    3$ r^  r   r_  s   r5   rY   re  .  r`  r8   c                  "   > ST R                    3$ rJ  r   rK  s   r5   rY   re  2  rL  r8   r   r   )
rJ   r\   r|   rR   rY  rZ  r[  r  r   r   r$  s   ``  r5    meta__weight_int4pack_mm_for_cpuri  (  s    	LLA>?	LLA>?	LL	EMM5==%..AA? 
LL	5;;8 ;;qvvay!&&)177;;;r8   r  r  c                     X-   S-
  U-  U-  $ r-  r=   rC  s     r5   kai_rounduprk  7  s    UQY1!!r8   c                   ^	^
^^^^^^^^^ U S:X  a{  X2:X  a(  SnSnSnSmSmSmS mUUUU4S jmU4S jnU" XXEU5      $ US-  S	:X  aD  X#-  S	:X  a;  SnSnSnSmSmSmSm	U	UU4S
 jnU	U
UUUUU4S jmS m
U	4S jmU	4S jmU" XXEXc5      $ g g g )Nr  rM  rX  r   c                 4    [        X-  S5      n[        X5      $ )Nr  rk  )r`  krsrkr_sr_roundedup4s       r5   kai_k_roundedup3get_kai_packed_weight_size.<locals>.kai_k_roundedupF  s     $/rw#: "177r8   c                 X   > T" XU5      nUS-  S:X  d   S5       eUUS-  T-   T-   T-   -  $ )Nr   r   zk_internal must be evenr=   )	r`  nrro  rp  
k_internalrr  kai_num_bytes_biaskai_num_bytes_multiplier_rhskai_num_bytes_sum_rhss	        r5   9kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0L  sW     -QB7
"Q1,G.GG,1_23+, )) r8   c                 8   > [        X5      U-  nUT" XX45      -  $ r2   rn  )r  r`  ru  ro  rp  num_rowsrz  s         r5   7kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0[get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0Z  s0     'q-3 Orr8   rN  r   c                 z   > XS-  S:X  d   eUT	-  S:X  d   eUT-  S:X  d   e[        X5      U-  nUT" XX4U5      -  $ rn  rn  )
r  r`  ru  ro  rp  blr}  kai_bl_multiple_of;kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0kai_nr_multiple_ofs
          r5   9kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0r  sh     A~%~//A555//A555&q-3 Qrrr8   c                    > XB-  S:X  d   eUT
-  S:X  d   eUT-  S:X  d   eT	" 5       nT" X5      nT" XE5      nUXv-  T-   T-   -  $ rn  r=   )r`  ru  ro  rp  r  num_bytes_multiplier_rhsnum_blocks_per_rownum_bytes_per_blockr  #kai_get_bf16_datatype_size_in_bytesr  kai_num_blocks_per_rowrw  kai_num_bytes_per_blockry  s           r5   r  _get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0  s     A~%~//A555//A555 ,O+P(%;A%B"&='# (=+,() r8   c                      g)Nr   r=   r=   r8   r5   r  Gget_kai_packed_weight_size.<locals>.kai_get_bf16_datatype_size_in_bytes  s    r8   c                 6   > UT-  S:X  d   e[        X5      U-  $ rn  rn  )r`  r  r  s     r5   r  :get_kai_packed_weight_size.<locals>.kai_num_blocks_per_row  s'    //A555"1)R//r8   c                 *   > U T-  S:X  d   eU S-  U-   $ )Nr   r   r=   )r  r  r  s     r5   r  ;get_kai_packed_weight_size.<locals>.kai_num_bytes_per_block  s'    //A555a#;;;r8   r=   )n_bitsr  K	groupsizekai_nrkai_krkai_srr~  r  r  r  r  rz  rr  r  r  rw  rx  r  ry  s            @@@@@@@@@@@r5   get_kai_packed_weight_sizer  ;  s    {>FFF$%!+,(!"8 
 Kff  ^q Q]a%7FFF$%!!"!"!#  ,0< Mff u &8 [ r8   c                 \  ^  [         R                  " T R                  [         R                  L U 4S j5        [         R                  R
                  R                  5       (       a  X4:X  a  UR                  [         R                  :X  d4  X4:  ac  US-  S:X  aZ  XC-  S:X  aR  UR                  [         R                  :X  a4  [        SXTU5      nT R                  [        U5      [         R                  S9$ T R                  5       UR                  5       -   nT R                  U[         R                  S9$ )Nc                  "   > ST R                    3$ rJ  r   )weightss   r5   rY   2meta__dyn_quant_pack_4bit_weight.<locals>.<lambda>  s    .w}}o>r8   rN  r   r  r   )rJ   r\   rR   r  backendskleidiaiis_availablerN   r[  r  r   r   r   )r  scales_zerosr:  
block_sizein_featuresout_featurespacked_weight_sizes   `      r5    meta__dyn_quant_pack_4bit_weightr    s     
LL$> ~~++--		"|'9'9U[['H$R1$(A-""enn4 8|*
   %7!8 LL <+=+=+??/u{{CCr8   c                   ^  [         R                  " T R                  5       S:H  S 5        [         R                  " T R                  [         R                  4;   U 4S j5        T R                  S5      nT R                  XTT R                  S9$ )Nr   c                      g)Nzinput must be a 2D tensorr=   r=   r8   r5   rY   -meta__dyn_quant_matmul_4bit.<locals>.<lambda>  s    )Dr8   c                  "   > ST R                    3$ )Nzexpected input to be f32, got r   )inps   r5   rY   r    s    0<r8   r   r   )rJ   r\   r|   rR   rY  r   r   )r  packed_weightsr  r  r  r  s   `     r5   meta__dyn_quant_matmul_4bitr    se     
LLa!DE	LL		emm_$< 	A==		=::r8   c                 0  ^ ^ [         R                  " T R                  5       S:H  S 5        [         R                  " T R                  [         R                  [         R
                  [         R                  4;   U 4S j5        [         R                  " TR                  5       S:H  S 5        [         R                  " TR                  [         R                  L U4S j5        T R                  T R                  S5      TR                  S5      T R                  S9$ )Nr   c                      grZ  r=   r=   r8   r5   rY   *meta__weight_int8pack_mm.<locals>.<lambda>  r=  r8   c                  "   > ST R                    3$ r^  r   r_  s   r5   rY   r    r`  r8   c                      grG  r=   r=   r8   r5   rY   r    r=  r8   c                  "   > ST R                    3$ )Nzexpected w to be int8, got r   rK  s   r5   rY   r    s    -aggY7r8   r   r   )
rJ   r\   r|   rR   rY  rZ  r[  r?  r   r   )rG   r  q_scaless   `` r5   meta__weight_int8pack_mmr    s    	LLA>?	LL	EMM5==%..AA? 
LLA>?	LL	5::7 ;;qvvay!&&)177;;;r8   c                   ^ ^^ [         R                  " T R                  5       S:  U 4S j5        [         R                  " TR                  5       S:  U4S j5        [         R                  " T R                  S5      TR                  S5      :H  U U4S j5        [         R                  " [        R
                  " T R                  5      S 5        [         R                  " [        R
                  " TR                  5      S 5        [         R                  " US:  S	 5        [         R                  " TS
;   U4S j5        T R                  S5      nTR                  S5      nT R                  S S nTR                  S S n[        [         R                  " Xg5      5      nUR                  XE/5        T R                  U5      $ )Nr   c                  ,   > ST R                  5        S3$ )Nz1cdist only supports at least 2D tensors, X1 got: r!  r   )x1s   r5   rY   $meta_cdist_forward.<locals>.<lambda>      CBFFH:QOr8   c                  ,   > ST R                  5        S3$ )Nz1cdist only supports at least 2D tensors, X2 got: r!  r   )x2s   r5   rY   r    r  r8   r   c                  P   > ST R                  S5       STR                  S5       3$ )Nz4X1 and X2 must have the same number of columns. X1: r   z X2: r   )r  r  s   r5   rY   r    s*    Frwwr{mSXY[Y`Y`acYdXefr8   c                      g)Nz=cdist only supports floating-point dtypes, X1 got: {x1.dtype}r=   r=   r8   r5   rY   r    r   r8   c                      g)Nz=cdist only supports floating-point dtypes, X2 got: {x2.dtype}r=   r=   r8   r5   rY   r    r   r8   r   c                      g)Nz)cdist only supports non-negative p valuesr=   r=   r8   r5   rY   r    s    !Lr8   Nr   r   c                     > ST  3$ )Nz%possible modes: None, 1, 2, but was: r=   )compute_modes   r5   rY   r    s    7~Fr8   r  )rJ   r\   r|   r   rB   is_float_dtyperR   r   r   broadcast_shapesextendr   )	r  r  rQ  r  r1r2batch_tensor1batch_tensor2rb  s	   `` `     r5   meta_cdist_forwardr    sF   	LL
AO 
LL
AO 
LL
rwwr{"f 
LLRXX&O 
LLRXX&O 
LLaLM	LL$F 
B	BHHSbMMHHSbMM..}LML!<<%%r8   c                 <   UR                   S   nUR                   S   nUR                   S   nUR                   S S nUR                   S S n	[        [        R                  " X5      5      n
U
R	                  5       nUR                  Xe/5        [        R                  " U
5      nUS:X  d  US:X  d  US:X  d  US:X  a  [        R                  " U5      $ U[        UR                   5      :w  a  UR                  U5      n[        R                  " U[        R                  S9$ )Nr   r  r   r   )r   r   rJ   r  copyr  mathprod
zeros_liker  r   r   )rV  r  r  rQ  cdistc1r  r  r  r  r  tensor1_expand_sizebatch_products                r5   meta_cdist_backwardr    s     
"B	"B	"BHHSbMMHHSbMM 6 6} TU.335x(II23M	Qw"'R1W(:##d288n,YY*+Be.E.EFFr8   c	                 Z  ^ ^^^^^ [         R                  " TR                  [         R                  [         R                  4;   U4S j5        [         R                  " TR                  [         R                  [         R                  4;   U4S j5        [         R                  " [
        R                  " T R                  5      U 4S j5        TR                  S5      n	U(       a   [         R                  " U	S:  S 5        U	S-  n	T R                  U	T R                  S5      5      n
Tb  [         R                  " U[        :H  S 5        [         R                  " TR                  S:H  U4S j5        [         R                  " TR                  5       TR                  5       :H  UU4S	 j5        U4S
 jmS mUU4S jn[        T5      S:w  a}  TR                  TR                  S5      5      nTR                  TR                  5       5      nU[        :X  a"  TR                  U	T R                  S5      5      nOTR                  S5      nOU" T TX5      nU[        [        4;   d  U(       d!  TR                  TR                  S5      5      nOTR                  S5      nTR                  U	5      nTR                  S   nU[        :X  aG  U(       a   [         R                  " US:  S 5        US-  nTR                  UT R                  S   5      nOTR                  UR                  5       5      nXX4$ )Nc                  "   > ST R                    3$ )Nz(expected indices to be long or int, got r   )r   s   r5   rY   $meta_embedding_bag.<locals>.<lambda>*      :7==/Jr8   c                  "   > ST R                    3$ )Nz(expected offsets to be long or int, got r   )rr  s   r5   rY   r  .  r  r8   c                  "   > ST R                    3$ )Nz/expected weight to be floating point type, got r   )r8  s   r5   rY   r  2  s    A&,,Pr8   r   r   c                      gNz1include_last_offset: numBags should be at least 1r=   r=   r8   r5   rY   r  9  s    Gr8   c                      g)Nz@embedding_bag: per_sample_weights only supported with mode='sum'r=   r=   r8   r5   rY   r  B  s    Vr8   c                  $   > ST R                    S3$ )Nz1expected per_sample_weights to be 1D tensor, got r!  r	  )per_sample_weightss   r5   rY   r  F  s    GHZH_H_G``abr8   c                  N   > STR                  5        ST R                  5        S3$ )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (rp   r   )r   r  s   r5   rY   r  J  s/    78J8P8P8R7S T66=mmo5FaIr8   c                 L   > T" XU5      =(       a    UR                  S5      S:H  $ Nr   r   r   )r(  r  r   padding_idxis_fast_path_index_selects       r5   is_fast_path_index_select_scale;meta_embedding_bag.<locals>.is_fast_path_index_select_scaleP  s&    %c;?XELLQROWXDX	
r8   c                    U R                   [        R                  :H  =(       d    U R                   [        R                  :H  =(       a;    U R	                  S5      S:H  =(       a     UR	                  S5      S:H  =(       a    US:  $ Nr   r   )rR   rJ   rN   rL   r   )r(  r   r  s      r5   r  5meta_embedding_bag.<locals>.is_fast_path_index_selectU  sb    YY%++%@ejj)@  

1" a A%  a		
r8   c                 .   > Ub	  T" XX#5      $ T" XU5      $ r2   r=   )r(  r  r   r  r  r  s       r5   is_fast_path(meta_embedding_bag.<locals>.is_fast_path]  s#    23vSS,S+FFr8   cpuc                      gr  r=   r=   r8   r5   rY   r  w  s    Or8   )rJ   r\   rR   r   r   rB   r  r   r   MODE_SUMr   r   r   MODE_MAX	MODE_MEANr   )r8  r   rr  scale_grad_by_freqrf  sparser  include_last_offsetr  num_bagsr   r  
offset2bagbag_sizemax_indicesfast_path_sumnumBagsr  r  s   ```   `          @@r5   meta_embedding_bagr     sw    
LL%**eii00J 
LL%**eii00J 
LLV\\*P
 ||AHMG	
 	AhA7F%HV	
 	##q(b	
 	$$&'--/9	



G 7u$&&w||A7
$$W\\^48!++Hfkk!nEK!++A.K$V-?UIx(( **7<<?;J **1-J$$X.--"8"qLO 1!++GV\\!_EK!++HMMO<Kx44r8   c                     [        XU/UQ76 u  pEpg[        U5      S:X  a  UR                  UR                  5       5      nXEXg4$ )Nr  )r   r   r   r   )r8  r   rr  rD   r   r  r  r  s           r5   meta_embedding_bag_forward_onlyr    sN    0B1#'1-F 7u$$$W\\^4x44r8   c                     U(       a  U$ U R                   R                  (       d  U R                   R                  (       a  U R                   $ U(       a  [        R                  $ U R                   $ r2   )rR   r   r   rJ   r   )r   rR   promote_int_to_longs      r5   _get_reduction_dtyper    sD    {{$$(>(>{{	zz;;r8   r   c                    [        XSS9n[        R                  " U R                  U5      n[	        XU5      nU R                  XTS9$ )NT)r  r   )r  rB   r  r   r  r   )r   r  r  rR   r  rb  s         r5   meta_nansumr    sC     ($OLT2D+EAL??<?<<r8   c           	          [         R                  " U R                  [        [	        U R                  5       5      5      5      nU R                  U5      $ r2   )rB   r  r   r[   r   r|   r   )r   rb  s     r5   meta_medianr	    s<    77U5-.L ??<((r8   c                    [        U 5      S:X  a  [        R                  " S5        [        R                  " U R                  U45      n[        XU5      nU R                  U5      U R                  U[        R                  S94$ )Nr   zmedian CUDA with indices outputr   )	r   rB   alert_not_deterministicr  r   r  r   rJ   r   )r   r|   r  rb  s       r5   meta_median_mode_dimr    sn     5V#%%&GH


u{{SF
3C+E@L%EJJ7 r8   c                     U $ r2   r=   r   s    r5   meta_logical_not_r    ry  r8   c                 V   [         R                  " [        U5      U R                  5       :  S 5        [        U5      U R                  5       -
  nSU-  [	        U R
                  5      -   n[        [        U5      5       Vs/ s H  oCU   X   -  PM     nnU R                  U5      $ s  snf )Nc                      g)NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr=   r=   r8   r5   rY   meta_repeat.<locals>.<lambda>  s    lr8   r  )rJ   r\   r   r|   r[   r   r   r   )r   r  num_new_dimensionspadded_sizer   target_sizes         r5   meta_repeatr    s    	LLG
"l W
2++eDJJ.??K8=c'l8KL8K1q>GJ.8KKL>>+&& Ms    B&c                     U $ r2   r=   r   s    r5   
meta_zero_r    ry  r8   c                     [        U[        R                  5      (       a   [        U R                  UR                  5        U $ r2   )rf   rJ   r   r^   r   r   r   s     r5   meta_binop_inplacer    s,     %&&

EKK8Kr8   c                 *   S nS nS nU" U 5      (       a  U" U5      (       a  [        S5      eU" U 5      (       a  U" U5      (       d  [        S5      e[        U[        R                  5      (       a   [	        U R
                  UR
                  5        U $ )a  
Some checks for inplace ops.
Checks for promotion rules for some dtypes.
int.add/sub_(float) and bool.add/sub_(others) are rejected.
Promoting in these in-place operations would require reallocating
and copying over elements, hence not allowed.
Checks for alpha param.
c                     [        U [        5      (       a   [        R                  " U R                  5      $ [        U [
        5      $ r2   )rf   r   rB   rD  rR   r   ri   s    r5   is_integeric.meta_binop_inplace_alpha.<locals>.is_integeric  s1    c:&&))#))44c7++r8   c                     [        U [        5      (       a   [        R                  " U R                  5      $ [        U [
        5      $ r2   )rf   r   rB   r  rR   r   r  s    r5   
is_floatic,meta_binop_inplace_alpha.<locals>.is_floatic   s1    c:&&''		22c9--r8   c                     [        U [        5      (       a   [        R                  " U R                  5      $ [        U [
        5      $ r2   )rf   r   rB   is_boolean_dtyperR   r   r  s    r5   is_booleanic.meta_binop_inplace_alpha.<locals>.is_booleanic  s1    c:&&))#))44c8,,r8   z]Promotion of int.add/sub_(float) in in-place ops are not possible due to element size change.z_Promotion of book.add/sub_(others) in in-place ops are not possible due to element size change.)r$  rf   rJ   r   r^   r   )r   r   rK  r  r!  r%  s         r5   meta_binop_inplace_alphar'    s    $,.- Dj//k
 	

 D,u"5"5m
 	
 %&&

EKK8Kr8   c                 2    [        U [        R                  S9$ Nr?   rH   r   rC   )r   kwargss     r5   
meta_roundr-    s    <DD r8   c                   ^ ^^ [         R                  " [        R                  " TR                  5      U U4S j5        [        T[         R                  5      (       a;  [         R                  " [        R                  " TR                  5      U U4S j5        g [         R                  " [        T[        5      U U4S j5        g )Nc                  &   > T  STR                    3$ )Nz7: Expected input tensor to have an integral dtype. Got r   )r  r   s   r5   rY   #shift_dtype_check.<locals>.<lambda>'  s    7)RSWS]S]R^_r8   c                  &   > T  STR                    3$ )Nz6: Expected shift value to have an integral dtype. Got r   r  r  s   r5   rY   r0  ,  s    wiUVYV_V_U`ar8   c                     > T  ST 3$ )Nz): Expected shift value to be an int. Got r=   r2  s   r5   rY   r0  1  s    wiHNr8   )rJ   r\   rB   rD  rR   rf   r   r   )r  r   r  s   ```r5   shift_dtype_checkr4  $  ss    	LLtzz*_ #u||$$""399-a	

 	sG$N	
r8   c                 J    [        SX5        [        X[        R                  S9$ )Nrshiftr*  r4  rH   r   rC   r  s     r5   meta_rshiftsr8  5  %    h,$C$K$K r8   c                 J    [        SX5        [        X[        R                  S9$ )Nlshiftr*  r7  r  s     r5   meta_lshiftsr<  =  r9  r8   c                 8    U R                  U R                  5      $ r2   r  r   s    r5   	meta_zeror>  E  s    >>$**%%r8   c                     U $ r2   r=   r   r  s     r5   
meta_fill_rA  J  ry  r8   c                 .    [         R                  " U 5      $ r2   r}  r@  s     r5   	meta_fillrC  O      D!!r8   c                     U $ r2   r=   r   s    r5   
meta_relu_rF  T  ry  r8   c                 2    [        X[        R                  S9$ r)  r+  )r   r   rK  s      r5   meta__add_relurH  Y  s     $C$K$K r8   c                 .    [         R                  " U 5      $ r2   r}  r   noiselowerr  r  r  s         r5   meta_rrelu_with_noiserM  a  s    
 D!!r8   c                 Z    [         R                  " U 5      [         R                  " U5      4$ r2   r}  rJ  s         r5    meta_rrelu_with_noise_functionalrO  i  s%     D!5#3#3E#:::r8   c                     U $ r2   r=   )r   rL  r  r  r  s        r5   meta_rrelu_with_noise_rQ  p  s	     Kr8   c                 .    [         R                  " U 5      $ r2   r}  r   r   r   
accumulates       r5   meta_index_putrU  w  rD  r8   c                 F    [        U R                  UR                  5        U $ r2   r^   r   )r   r  values      r5   meta_masked_fill_rY  |  s    DJJ

3Kr8   c                     U R                  U R                  5       5      R                  [        R                  " U 5      S9nU$ r   )r   r   r%  rB   r  )r   r  r  masked_scales       r5   meta__masked_scaler\    s<    >>$))+.1111$7 2 L r8   c                    ^ ^ [         R                  " UR                  [         R                  [         R                  4;   S 5        [         R                  " T R                  TR                  :H  U U4S j5        T $ )Nc                      g)NzMask must be bool or uint8r=   r=   r8   r5   rY   &meta_masked_scatter_.<locals>.<lambda>  s    9Ur8   c                  <   > ST R                    STR                    3$ )NzEmasked_scatter: expected self and source to have same dtypes but got r   r   )r   re  s   r5   rY   r_    s      **U6<<.:r8   )rJ   r\   rR   r  r  )r   r  re  s   ` `r5   meta_masked_scatter_ra    sU    	LL

uzz5;;//1U 
LL

fll"	:
 Kr8   c                 z    [        X5      u  p[        R                  " U [        R                  S9n[	        X1U5      $ r   )r%   rJ   r   r   ra  )r   r  re  r   s       r5   meta_masked_scatterrc    s5     "$-JDd%2I2IJFf55r8   c                 $    U R                  U5      $ r2   r  )r   r  rN  s      r5   meta_masked_scatter_backwardre    s    >>%  r8   c                     U $ r2   r=   rS  s       r5   meta_index_put_rg    ry  r8   c                 8    U R                  U R                  5      $ r2   )viewr   r   s    r5   
meta_aliasrj    s    99TZZ  r8   c                 j  ^^^	^
^ [         R                  " U R                  5       S:H  S 5        [         R                  " UR                  5       S:H  S 5        U R                  5       nUR                  5       mUS   m	US   m
US   nTS   nT	XV4m[         R                  " TS   T	:H  =(       a    TS   T
:H  UU	U
4S j5        UR	                  T5      nU(       dY  TbV  [         R                  " TR                  5       S:H  S 5        [         R                  " TR                  5       T:H  UU4S	 j5        U$ )
Nr-   c                      gr_  r=   r=   r8   r5   rY   )common_meta_baddbmm_bmm.<locals>.<lambda>  ra  r8   c                      grc  r=   r=   r8   r5   rY   rm    ra  r8   r   r   r   c            	      .   > ST ST ST S    ST S    S3	$ rh  r=   rj  s   r5   rY   rm    s3    RSURV
l<?*;2l1o=NbRr8   c                      g)Nzself must be a 3D tensorr=   r=   r8   r5   rY   rm    s    6Pr8   c                  0   > ST  STR                  5        3$ )Nz*Expected an input tensor shape with shape z but got shape: r   )r  self_baddbmms   r5   rY   rm    s    @M]^j^o^o^q]rsr8   )rJ   r\   r|   r   r   )re  rf  is_bmmrr  rq  res_rowsres_colsr   rk  rl  rm  r  s      `    @@@@r5   common_meta_baddbmm_bmmrv    s   	LL"$HI	LL"$HI;;=L;;=L	aB#AAHAHx*K	LLQ2E,q/5E"E	R k*Fl.\%%'1,.PQ;.s	

 Mr8   c                     [        XS5      $ )NT)rv  )r   rG  s     r5   meta_bmmrx    s    "4t44r8   c                 h    X-  nX-  nUS:w  a#  [        US:  5      [        US:  5      :w  a  US-  nU$ r  )r  )rG   yqrj  s       r5   div_rtnr|    s>    	A	A 	Av4A;$q1u+-	QHr8   c                     [        U U-   U-   XQS-
  -  -
  S-
  U(       a  US-
  OS-   U5      S-   nU(       a  US-
  U-  X-   :  a  US-  nU$ r  )r|  )	inputSize
kernelSizer  r  r   r  r  
outputSizes           r5   pooling_output_shape_pad_lrr    s     	 q.)* 	
 'vzA/ 	
 		  Nf$	(99!OJr8   c           	         ^^^ [         R                  " US:g  S 5        [         R                  " TS:  U4S j5        [         R                  " TTS-
  T-  S-   S-  :*  UUU4S j5        [        U TTTUTU5      $ )Nr   c                      g)Nzstride should not be zeror=   r=   r8   r5   rY   &pooling_output_shape.<locals>.<lambda>  s    &Ar8   c                     > ST  3$ )Nz'pad must be non-negative, but got pad: r=   )pads   r5   rY   r    s    %LSE#Rr8   r   r   c                     > ST ST ST  3$ )NzApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r=   )r  r  r  s   r5   rY   r    s"    OPSu U%,nXJ@r8   )rJ   r\   r  )r~  r  r  r   r  r  s    `` ` r5   rO  rO    ss    	LL1AB	LLRS	LLa8+a/A55	
 ':sC9 r8   c           	        ^ ^^^^^	^
^^^^ T R                  5       nT	m[        R                  " TS:  =(       a    TS:  S 5        [        R                  " US:  =(       a    US:  S 5        [        R                  " US:  =(       a    US:  S 5        T R                  S5      S:g  =(       a    T R                  S5      S:g  nU[        R                  :X  a@  [        R                  " US:H  =(       a    U=(       a    T R                  S5      S:g  S	 5        Or[        R                  " US:H  =(       a    T R                  S5      S:g  =(       a    U=(       d)    US:H  =(       a    U=(       a    T R                  S5      S:g  U 4S
 j5        [        R                  " TS-  T:  =(       a    TS-  T:  UUUU4S j5        [        R                  " TS:  =(       a    TS:  U
UU	UUU4S j5        g )Nr   c                      g)NzCkernel size should be greater than zero, but got kH: {kH}, kW: {kW}r=   r=   r8   r5   rY   $pool2d_shape_check.<locals>.<lambda>  r  r8   c                      g)Nz>stride should be greater than zero, but got dH: {dH}, dW: {dW}r=   r=   r8   r5   rY   r  "  s    Pr8   c                      g)Nz\dilation should be greater than zero, but got dilationH: {dilationH}, dilationW: {dilationW}r=   r=   r8   r5   rY   r  &  s    nr8   r   r   r  r-   c                      g)NzExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: {input.size()}r=   r=   r8   r5   rY   r  .  s     Qr8   c                  *   > ST R                  5        3$ )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: r   rF  s   r5   rY   r  5  s    opupzpzp|o}~r8   c                      > ST ST ST ST  3$ )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r=   )rS  rT  rW  rX  s   r5   rY   r  :  s$     ygbT>r8   c                  .   > ST ST  ST ST ST ST S3$ NzGiven input size: (rG   z). Calculated output size: (z). Output size is too smallr=   )rY  rZ  r  r_  r  r  s   r5   rY   r  @  s8    %k]!K=* N$$0><.+ O##r8   )r|   rJ   r\   r   r  )r   rS  rT  rU  rV  rW  rX  	dilationH	dilationWr  rY  rZ  r  r  r   r   
valid_dimsr_  s   ```  ``  `````   @r5   rP  rP    s   " 99;DL	LL
Q26U 
LL
Q26P 
LLA')a-n
 A!#:

1(:J+++AI;*;A!);Q	
 	QY<5::a=A-<* A	?j?UZZ]a-?~	
 
LL
a4+B!GtO	> 
LLq.\Q.	# 	#r8   r  r}  rS  rT  r~  rU  rV  pTpHpW	dilationTr  r  r  r  r  r  r  r  r{  c           
        ^ ^^^^^^^^^	^
^^^^^^^^^^^ T R                   n[        R                  " TS:  =(       a    TS:  =(       a    TS:  UUU4S j5        [        R                  " TS:  =(       a    TS:  =(       a    TS:  UUU4S j5        [        R                  " TS:  =(       a    TS:  =(       a    TS:  UUU4S j5        [        R                  " US;   UU 4S j5        [        U5       H@  mUS:X  a  TS:X  a  M  [        R                  " T R	                  T5      S:  UUU 4S j5        MB     U(       a;  [        R                  " TT:  =(       a    TT:  =(       a    TT:  UUUUUU4S	 j5        [        R                  " TS
-  T:  =(       a    TS
-  T
:  =(       a    TS
-  T	:  UUUU	UU
4S j5        [        R                  " TS:  =(       a    TS:  =(       a    TS:  UUUUUUU4S j5        g )Nr   c                     > ST ST  ST 3$ )Nz5kernel size should be greater than zero, but got kT: z, kH: z, kW: r=   )rS  r}  rT  s   r5   rY   $pool3d_shape_check.<locals>.<lambda>b  s    $fRDrd,r8   c                     > ST ST  ST 3$ )Nz0stride should be greater than zero, but got dT: z, dH: z, dW: r=   )rU  r~  rV  s   r5   rY   r  i  s    >rd&FSURVWr8   c                     > ST ST  ST 3$ )Nz9dilation should be greater than zero, but got dilationT: z, dilationH: z, dilationW: r=   )r  r  r  s   r5   rY   r  o  s    #M)M)Vr8   rv  c                  &   > T  STR                    3$ )Nz/: Expected 4D or 5D tensor for input, but got: r?  )r  r   s   r5   rY   r  w  s    7)J5;;-Xr8   r6  c                  L   > T  STR                    STR                  T5       S3$ )NzZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)r   r   )r  r   r   s   r5   rY   r    s.    ) --2[[M+EJJqM?:KMr8   c                  .   > ST ST  ST ST ST ST S3$ )Nzinput image (T: r?  r"  z ) smaller than kernel size (kT:  kH:  kW: rp   r=   )r  r  r  rS  r}  rT  s   r5   rY   r    s4    "5'gYd6( C$$&4uRDbT<r8   r   c                  ,   > ST ST ST  ST ST ST 3$ )NzHpad should be smaller than or equal to half of kernel size, but got kT: r  r  z padT: z padW: z padH: r=   )rS  r}  rT  r  r  r  s   r5   rY   r    s1    $eB4uRDt72$gbTKr8   r   c                  :   > ST ST ST  ST ST ST ST ST S3$ r  r=   )r  r  r  r  r  r  r  s   r5   rY   r    sD    !'!E7!G9AfX F((/y%'!F8 L'(r8   )r   rJ   r\   r   r   )r   r  r}  rS  rT  r~  rU  rV  r  r  r  r  r  r  r  r  r  r  r  r  r  r{  r   r   s   `````````````````````  @r5   r|  r|  F  s   0 ::D	LL
Q$26$b1f	
 
LL
Q$26$b1f	
 
LLA9)a-9IM	
 
LLX
 4[19aJJqMA	
	  RK:GrM:fl 	
 
LL
Q"6a26"q&B,	
 	
 
LL
3v{3w!|	
 	
r8   c                 j   U R                   n[        U UUUUUUU	U
UUUUUUUUUUUU5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        g )Nr  r-   r   r   r   r|  r-  )r   r  r   r  r}  rS  rT  r~  rU  rV  r  r  r  r  r  r  r  r  r  r  r  r  r  r   s                           r5   max_pool3d_backward_shape_checkr    s    2 ::D








+0 ;dQh8;dQh6;dQh8;dQh77D$(G47D$(E27D$(G47D$(F3r8   c                     U R                   n[        U UUUUUUUU	U
USSSUUUUUUUS5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        [        UUUS-
  U5        g )Nr   Tr  r-   r   r  )r   r  r  r}  rS  rT  r~  rU  rV  r  r  r  r  r  r  r  r  r  r  r   s                       r5   r  r    s    * ::D








			-2 ;dQh8;dQh6;dQh8;dQh7r8   c                    S nU" SU5      u  px[         R                  " [        U5      S;   S 5        [        U5      S:X  a  XxpOU" SU5      u  pU" SU5      u  pU" SU5      u  pU R                  S	5      nU R                  S
5      nU R                  S5      n[        R
                  " U 5      nU[         R                  :X  a*  [         R                  " U R                  5       S:H  S 5        OVU[         R                  :X  a*  [         R                  " U R                  5       S;   S 5        O[         R                  " SS 5        [        UX{XU5      n[        UXXU5      n[        U UUU	U
UUUUUUUUUU5        UUU4$ )Nc                    ^  [         R                  " [        U5      S;   U 4S j5        US   n[        U5      S:X  a  UOUS   nX#4$ )Nr0  c                     > ST  S3$ )Nzmax_pool2d: r2  r=   r3  s   r5   rY   Emax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>"  r5  r8   r   r   r6  r7  s   `   r5   r:  3max_pool2d_checks_and_compute_shape.<locals>.unpack  r<  r8   r  r=  c                      g)NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr=   r=   r8   r5   rY   5max_pool2d_checks_and_compute_shape.<locals>.<lambda>,  rA  r8   r   r   r  r  rK  r  r   r  c                      g)NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr=   r=   r8   r5   rY   r  =  s    cr8   r  c                      g)Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr=   r=   r8   r5   rY   r  B      Or8   Fc                      g)Nz?Unsupport memory format. Supports only ChannelsLast, Contiguousr=   r=   r8   r5   rY   r  G  s    Ur8   )rJ   r\   r   r   rB   r  r  r|   r   rO  rP  )r   r  r   r  r  r  r:  rS  rT  rU  rV  rW  rX  r  r  r  rY  rZ  r   r  r  s                        r5   r  r    sy    M;/FB	LLFy a 6{aB&)	7+JD!*h7I**R.K**R.KBJ//6M+++IIK1c	
 
%11	1IIK6!O	

 	U	

 (RriXL&z2RIVK



$ k11r8   c                 |  ^ ^^^^^ [        TX#XEU5      u  nmm[        R                  " TR                  T R                  :H  U U4S j5        UmTR                  mUUUU4S jn	U	" T 5        U	" U5        [
        R                  " T5      n
[        R                  " TR                  TR                  TR                  U
S9$ )Nc                  <   > STR                    ST R                    3$ )NzExpected dtype z  for `gradOutput` but got dtype r   r  s   r5   rY   7meta_max_pool2d_with_indices_backward.<locals>.<lambda>w  s    /$**-MkN_N_M`ar8   c                 l   > [        U TTS-
  T5        [        U TTS-
  T5        [        U TTS-
  T5        g )Nr-   r   r   )r-  )r  r_  r   r  r  s    r5   _check_dim_size>meta_max_pool2d_with_indices_backward.<locals>._check_dim_size}  s9    q$q,7q$q,7q$q+6r8   ry  )
r  rJ   r\   rR   r   rB   r  r   r   ry   )r  r   r  r   r  r  r  r   r  r  r   r_  r   r  r  s   ``         @@@@r5   %meta_max_pool2d_with_indices_backwardr  b  s     	,k7i		
 
LL

k'''a
 L99D7 7
 K G//5M;;

jj{{#	 r8   c                    [        XX#XE5      u  nnnU R                  5       S:X  a  U R                  S5      OSn	[        R                  " U 5      n
U R                  5       S:X  a  XgU/nOXXx/n[
        R                  " UU R                  U R                  U
S9[
        R                  " U[
        R                  U R                  U
S94$ r  )
r  r|   r   rB   r  rJ   r   rR   ry   r   r  s               r5   meta_max_pool2d_with_indicesr    s     	,FX		
  %yy{a/UZZ^QF//6Myy{a;7\?++<<'		
 	++<<'		
 r8   c           	        ^ ^^^
^^ [         R                  " T R                  S;   U 4S j5        T R                  n[        US-
  U5       HA  m
[         R                  " T R	                  T
5      S:  ST R	                  5        ST
 S35        MC     [         R                  " [        T5      S:H  S	 5        [         R                  " [        U5      S:H  S
 5        T R	                  S5      nT R	                  S5      mT R	                  S5      mUS:X  a  T R	                  S5      nOSn[         R                  " T R                  TR                  :H  S 5        [         R                  " TR                  S:H  U4S j5        TR	                  S5      nTR	                  S5      nTR	                  S5      m
[         R                  " Xv:  S5        [         R                  " X:H  S 5        [         R                  " T
S:H  U
4S j5        [         R                  " US   TS   -   S-
  T:*  UU4S j5        [         R                  " US   TS   -   S-
  T:*  UU4S j5        T R                  5       S:X  a  XeUS   US   /n	O
XRS   US   /n	[         R                  " U	T R                  T R                  S9[         R                  " U	[         R                  T R                  S94$ )Nr  c                  "   > ST R                    3$ )Nz:fractional_max_pool2d: Expected 3D or 4D tensor, but got: r	  r   s   r5   rY   ,meta_fractional_max_pool2d.<locals>.<lambda>  s    LTYYKXr8   r-   r   z^fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimenions, but got r  z emptyr   c                      g)NzNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr=   r=   r8   r5   rY   r         2r8   c                      g)NzOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr=   r=   r8   r5   rY   r    r  r8   rK  r  r   r  r   c                      g)Nz6Expect _random_samples to have the same dtype as inputr=   r=   r8   r5   rY   r    s    Hr8   c                  "   > ST R                    3$ )Nz1Expect _random samples to have 3 dimensions got, r	  )random_sampless   r5   rY   r    s    CNDWDWCXYr8   z=Expect _random_samples.size(0) no less then input batch size.c                      g)Nz<Expect _random_samples.size(1) equals to input channel size.r=   r=   r8   r5   rY   r        Nr8   c                     > ST  S3$ )Nz/Expect _random_samples.size(2) equals to 2 got .r=   )r   s   r5   rY   r    s    #RSTRUUV!Wr8   c                     > STS    ST  3$ )Nz%fractional_max_pool2d: kernel height r   z' is too large relative to input height r=   )input_heightr  s   r5   rY   r    s    7A7GGno{n|}r8   c                     > STS    ST  3$ )Nz$fractional_max_pool2d: kernel width r   z& is too large relative to input width r=   )input_widthr  s   r5   rY   r    s    6{1~6FFlmxlyzr8   r  )rJ   r\   r   r   r   r   rR   r|   r   ry   r   )r   r  r  r  r   input_channelsinput_batchr  cr   r   r  r  s   `` `      @@@r5   meta_fractional_max_pool2dr    s   	LL		VX 99D4!8T"IIaL166:iik]BRSTRUU[]	
 # 
LLKA	2
 
LLKA	2 YYr]N99R=L))B-Kqyiil	LL

n***H 
LLq Y
 	AAAAAA	LL	G 
LL	N 
LLaWX	LLAQ'!+|;} 
LLAQ'!+{:z
 xxzQ[^[^LAA? 	**;;	

 	++;;	
 r8   c                    [         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " U(       + =(       d    [        U5      S;   S 5        U(       d  UOUS   n	U(       d  UO[        U5      S:X  a  U	OUS   n
U(       d  UO[        U5      S:X  a  U	OUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " U R                  S	;   S
 5        U R                  S:X  a  U R	                  S5      OSnU R	                  S5      nU R	                  S5      nU R	                  S5      nU R	                  S5      n[        UXlXU5      n[        UX}U
UU5      n[        UXUUU5      n[        U UUUUU	U
UUUUUUUUUUUUUS5        U R                  S:H  =(       a'    [        R                  " U 5      [         R                  :H  nU R                  S:X  aQ  U R                  S5      nUR                  5       (       + =(       a    UR                  [         R                  S9nUUUU4nOUUUUU4nU R                  U5      nU R                  U[         R                  S9nU(       a:  UR                  [         R                  S9nUR                  [         R                  S9nUU4$ )Nrk  c                      gNzMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr=   r=   r8   r5   rY   .meta_max_pool3d_with_indices.<locals>.<lambda>      _r8   r   r   r   c                      gNzQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr=   r=   r8   r5   rY   r        cr8   c                      gNzImax_pool3d: padding must either be a single int, or a tuple of three intsr=   r=   r8   r5   rY   r  &      [r8   c                      gNzJmax_pool3d: dilation must be either a single int, or a tuple of three intsr=   r=   r8   r5   rY   r  .  r  r8   rv  c                      grx  r=   r=   r8   r5   rY   r  6  ry  r8   r6  r  rK  r  r   zmax_pool3d_with_indices()r  r   r   )rJ   r\   r   r   r   rO  r|  rB   r  r  r  r]  r   r   r%  )r   r  r   r  r  r  r}  rS  rT  r~  rU  rV  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  input_channels_last_checkr   r   r   s                                  r5   meta_max_pool3d_with_indicesr  
  sB    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
+c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
  %zzQUZZ^AFjjnGJJrNEjjnGZZ^F yIE"7BB	9MG!&""iKF








#+2 	

aXE77>%BXBXX  zzQ$)OOA$6!)7799
'5500 6 
 	
 eWf5	WeWf=	
//)
$Cooiu{{o;Gff5#9#9f:**5+A+A*B<r8   c                 &   [         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n	[        U5      S:X  a  UOUS   n
[         R                  " U(       + =(       d    [        U5      S;   S 5        U(       d  UOUS   nU(       d  U	O[        U5      S:X  a  UOUS   nU(       d  U
O[        U5      S:X  a  UOUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " [        U5      S;   S 5        US   n[        U5      S:X  a  UOUS   n[        U5      S:X  a  UOUS   n[         R                  " UR                  S	;   S
 5        UR	                  S5      nUR	                  S5      nUR	                  S5      nUR	                  S5      nU R	                  S5      nU R	                  S5      nU R	                  S5      n[        UU UUUU	U
UUUUUUUUUUUUUUUS5        UR                  S:H  =(       a'    [        R                  " U5      [         R                  :H  nUR                  S:X  aJ  UR                  S5      nUR                  5       (       + =(       a    UR                  [         R                  S9nUR                  UR                  5      nU(       a  UR                  [         R                  S9nU$ )Nrk  c                      gr  r=   r=   r8   r5   rY   7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r  r8   r   r   r   c                      gr  r=   r=   r8   r5   rY   r    r  r8   c                      gr  r=   r=   r8   r5   rY   r    r  r8   c                      gr  r=   r=   r8   r5   rY   r    r  r8   rv  c                      grx  r=   r=   r8   r5   rY   r    ry  r8   r  rK  r  r   z"max_pool3d_with_indices_backward()r6  r  r   )rJ   r\   r   r   r   r  rB   r  r  r  r]  r   r   r%  )r  r   r  r   r  r  r  r   r}  rS  rT  r~  rU  rV  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                                 r5   %meta_max_pool3d_with_indices_backwardr  s  s    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
+c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
 jjnGJJrNEjjnGZZ^FR Er"Gb!F#








,/6 	

aXE77>%BXBXX  zzQ$)OOA$6!)7799
'5500 6 
 	 -J]]1G1G]H
r8   gridc                   ^ ^^ [         R                  " T R                  TR                  :H  UU 4S j5        [         R                  " T R                  [         R                  :H  =(       a    TR                  [         R                  :H  UU 4S j5        [         R                  " T R
                  S   TR
                  S   :H  UU 4S j5        [         R                  " TR
                  S   T R                  S-
  :H  UU 4S j5        [        ST R                  5       H/  m[         R                  " T R
                  T   S:  UU 4S j5        M1     g )	Nc                  <   > STR                    ST R                    3$ )NzNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on r  r  r   s   r5   rY   +check_grid_sampler_common.<locals>.<lambda>  s"    \\N"24;;-Ar8   c                  <   > STR                    ST R                    3$ )NzTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )rx   r  s   r5   rY   r    s!    nT[[MCr8   r   c                  <   > STR                    ST R                    3$ )NzZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes  and grid with sizes r?  r  s   r5   rY   r    s"      %},A$**Or8   r   r   c                  B   > STR                   S-
   ST R                   3$ )Nz+grid_sampler(): expected grid to have size r   z, in last dimension, but got grid with sizes )r   r   r  s   r5   rY   r    s'    9%**q.9I J226**?r8   c                  *   > STR                    ST  S3$ )NzYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes r  r  r?  r  s   r5   rY   r    r  r8   )rJ   r\   ry   rx   r  r   r   r   )r   r  r   s   ``@r5   check_grid_sampler_commonr    s    	LL#	
 
LL%F$++*F	
 
LLA$**Q-'	
 
LL

2%**q.(	
 1ejj!KKNQ	
 "r8   c                        \ rS rSrSrSrSrSrg)GridSamplerInterpolationi  r   r   r   r=   N)rr   
__module____qualname____firstlineno__BILINEARNEARESTBICUBIC__static_attributes__r=   r8   r5   r  r    s    HGGr8   r  interpolation_modec                 6  ^ ^ [         R                  " T R                  S:H  =(       a    T R                  TR                  :H  UU 4S j5        [         R                  " T R                  S:H  =(       a    U[        R                  R
                  :H  (       + S 5        g )Nr6  c                  <   > STR                    ST R                    3$ )Nzdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes r  r?  r  s   r5   rY   'check_grid_sampler_3d.<locals>.<lambda>  s!    449KK=#DJJ<1r8   c                      g)Nz<grid_sampler(): bicubic interpolation only supports 4D inputr=   r=   r8   r5   rY   r    r  r8   )rJ   r\   r   r  r  rX  )r   r  r  s   `` r5   check_grid_sampler_3dr    sp    	LL

a3EJJ$))3	
 
LLJJ!O M"&>&F&F&L&LL	
 	Or8   c                     US   nU(       a$  [         R                  " U[         R                  S9nOS n[         R                  " U[         R                  S9n	X4$ Nr   r   )rJ   r  r   r   
r  r   r  r  padding_modealign_cornersr  input_requires_gradr  	grad_grids
             r5   grid_sampler_2d_backward_metar    sO     &a.%%e5;R;RS

  U5L5LMI""r8   c                     [        X5        [        XU5        U R                  S   nU R                  S   nUR                  S   nUR                  S   nUR                  S   n	U R                  XVXxU	45      $ )Nr   r   r   r-   )r  r  r   r   )
r   r  r  r  r	  r  Cout_Dout_Hout_Ws
             r5   grid_sampler_3dr  -  sn     e*%'9:AAAAJJqMEJJqMEJJqME??A%677r8   r  c                     [        X5        [        XU5        US   nU(       a$  [        R                  " U[        R                  S9nOS n[        R
                  " U[        R                  S9n	X4$ r  )r  r  rJ   r  r  r   r  s
             r5   grid_sampler_3d_backwardr  @  sg     e*%'9:%a.%%!?!?

 
  U5S5STI  r8   c                     UR                  SS 5      nU(       d  [        R                  " U5      nXCS'   [        R                  " U /UQ70 UD6$ )NrR   )rQ   rB   	get_dtyperJ   r   )r   r  rD   r,  rR   s        r5   fullr  X  sC    JJw%E
+7O;;t-d-f--r8   c           	      V   U[         R                  :X  a  [         R                  " US L S 5        [         R                  " SUc  U R                  OUUUc  U R
                  OUUS9nU R                  (       a>  UR                  U R                  5       U R                  5       U R                  5       5        O/UR                  U R                  5       U R                  5       S5        UR                  S5        U$ [        R                  R                  U UUUUUS9nUR!                  S5        U$ )Nc                      g)Nz9memory format option is only supported by strided tensorsr=   r=   r8   r5   rY   zeros_like.<locals>.<lambda>n  r  r8   r   r  Tr  )rJ   
sparse_coor\   r   rR   ry   	is_sparsesparse_resize_and_clear_r   
sparse_dim	dense_dimr|   _coalesced_r*   r   r'  fill_)r   rR   rx   ry   rz   r   r  s          r5   r  r  b  s     !!!T!O	

 kk %$**5"(.4;;f!
 >>((		T__.0@ ((dhhj!D

//
!
!# " C IIaLJr8   c                   ^ ^^ SSK Jn  T R                  5       n[        R                  " US:g  S 5        TS:  a  TOTU-   mT R                  T5      n[        R                  " U" T* U:  5      =(       d    U" TU:  5      (       + UUU 4S j5        TS:  a  TOTU-   m[        T R                  5       5      n[        T R                  5       5      nT R                  5       TUT   -  -   nUT	 UT	 T R                  XgU5      $ )Nr   r  c                      g)Nz-select() cannot be applied to a 0-dim tensor.r=   r=   r8   r5   rY   meta_select.<locals>.<lambda>  s    ?r8   c                  6   > ST STR                  5        ST  3$ )Nzselect(): index z! out of range for tensor of size z at dimension r   r|   r   r   s   r5   rY   r$    s#    "5')J99;-~cU,r8   )
r"  r  r|   rJ   r   r   r   r   r   rA  )	r   r|   r   r  r   r   new_sizer2  new_storage_offsets	   ```      r5   meta_selectr)    s    J88:D		?
 #sTzC99S>D	 %$/V3GQU3V	
	,	 aZEUT\EDIIK Hdkkm$J,,.C1HH3??81CDDr8   c                 .    [         R                  " U 5      $ r2   rB   clone_preserve_strides)r   r(  r|   r   s       r5   meta_select_scatterr-        ''--r8   c                 .    [         R                  " U 5      $ r2   r+  )r   r(  r|   rt   rs   steps         r5   meta_slice_scatterr1    r.  r8   dim_post_exprwrap_scalarc                     US::  a  U(       d   eSnU* nUS-
  nX:  d  X:  a   SU  SU SU S35       eU S:  a  X-  n U $ )Nr   r   zdim z out of bounds (ro   rp   r=   )r|   r2  r3  r   r  s        r5   r   r     sg    {.C
!
C	SYR4u4DSEC5PQ)RR'
QwJr8   c                 L    U R                  5       S:X  a  S$ U R                  U   $ r  r*  )r  r|   s     r5   ensure_nonempty_sizer6    s!    11.!''#,.r8   c                 F  ^ ^^^ [        T R                  5       S5      n[        TR                  5       S5      n[        R                  " X4:H  S 5        [	        U5       H@  mTT:w  d  M  [        R                  " [        TT5      [        T T5      :*  UUUU 4S j5        MB     g )Nr   c                      g)NzDIndex tensor must have the same number of dimensions as input tensorr=   r=   r8   r5   rY   $gather_shape_check.<locals>.<lambda>  s    Vr8   c                  N   > ST STR                    3STR                    ST  3-   $ )Nz!Size does not match at dimension z expected index  to be no larger than self  apart from dimension r?  )r|   r   r   r   s   r5   rY   r9    s5    ;A3>Nu{{m\/

|;QRUQVWXr8   )r  r|   rJ   r\   r   r6  )r   r|   r   	self_dims
index_dimsr   s   ```  @r5   gather_shape_checkr?    s}    DHHJ"IUYY[!$J	LLV 98LL$UA.2FtQ2OOX r8   c                 <  ^ SSK Jn  [        XR                  5       5      nU" TR	                  5       S:H  5      nU(       dB  [
        R                  " TR                  [
        R                  :H  U4S j5        [        XT5        U R                  TR                  5      $ )Nr   r  c                  "   > ST R                    3$ )Nz2gather(): Expected dtype int64 for index, but got r   r   s   r5   rY   meta_gather.<locals>.<lambda>  s    HVr8   )r"  r  r   r|   r   rJ   r\   rR   r   r?  r   r   )r   r|   r   sparse_gradr  wrapped_dimis_index_emptys     `    r5   meta_gatherrF    sl    J hhj1K)%++-1*<=NKK5::%V	
 	4e4>>%++&&r8   c                     U(       a<  U S:X  a  gU S:X  a  gU S:X  a  gU S:X  a  gU S	:X  a  g
[         R                  " SS 5        g U S:X  a  gU S:X  a  g[         R                  " SS 5        g )Nr  
REDUCE_ADDr  REDUCE_MULTIPLYmeanREDUCE_MEANamaxREDUCE_MAXIMUMaminREDUCE_MINIMUMFc                      g)Nz=reduce argument must be either sum, prod, mean, amax or amin.r=   r=   r8   r5   rY   #get_operator_enum.<locals>.<lambda>  s    Sr8   addmultiplyc                      g)Nz/reduce argument must be either add or multiply.r=   r=   r8   r5   rY   rQ    s    $Ur8   rm  )reduce_use_new_optionss     r5   get_operator_enumrW    s{    e$ ##S	
 	e
"$UUVr8   c                 &  ^  SSK Jn  U" UR                  5       S:g  5      (       a6  [        R                  " UR
                  [        R                  :H  U 4S j5        Ub3  [        R                  " UR
                  UR
                  :H  U 4S j5        g g )Nr   r  c                     > T  S3$ )Nz"(): Expected dtype int64 for indexr=   method_names   r5   rY   ,scatter_gather_dtype_check.<locals>.<lambda>  s    {m#EFr8   c                     > T  S3$ )Nz0(): Expected self.dtype to be equal to src.dtyper=   rZ  s   r5   rY   r\    s    {m#STr8   )r"  r  r   rJ   r\   rR   r   )r[  r   r   src_optr  s   `    r5   scatter_gather_dtype_checkr_    sg    JEKKMQ.//KK5::%F	

 JJ'--'T	
 r8   c                     [        U S5      $ r-  )r  r   s    r5   ensure_nonempty_dimra    s    sA;r8   c                 .  ^ ^^^ SSK Jn  U" TR                  5       S:H  5      (       a  g [        R                  " [        T R                  5       5      [        TR                  5       5      :H  S 5        Sn[        T R                  5       5      n[        U5       H+  n[        TU5      nUT:X  a  M  U[        T U5      :  d  M)  Sn  O   U(       d5  Tb2  [        U5       H#  n[        TU5      nU[        TU5      :  d  M!  Sn  O   Tbm  [        R                  " [        T R                  5       5      [        TR                  5       5      :H  S 5        [        R                  " U(       + UUU U4S j5        g [        R                  " U(       + UUU 4S j5        g )	Nr   r  c                      gNzCIndex tensor must have the same number of dimensions as self tensorr=   r=   r8   r5   rY   %scatter_shape_check.<locals>.<lambda>#  r  r8   FTc                      grd  r=   r=   r8   r5   rY   re  =  s    Yr8   c                  b   > STR                    STR                    3ST  STR                    3-   $ )NExpected index r;  r<  z and to be no larger than src r?  )r|   r   r   r^  s   r5   rY   re  A  s6    oekk]2Mdjj\Z&se+I'--YZr8   c                  H   > STR                    STR                    3ST  3-   $ )Nrh  r;  r<  r?  r&  s   r5   rY   re  G  s*    oekk]2Mdjj\Z&se,-r8   )	r"  r  r   rJ   r\   ra  r|   r   r6  )	r   r|   r   r^  r  is_wrong_shaper=  r   index_d_sizes	   ````     r5   scatter_shape_checkrl    sI   JEKKMQ.//	LLDHHJ'+>uyy{+KKU
 N#DHHJ/I 9+E158.tQ77!N  g1y!A/q9L27A>>!%	 " 
+/B599;/OOY	
 	Z	
 	-	
r8   c                     [        XR                  5       5      n[        SXU5        [        XX#5        Ub  [	        XE5        g g )Nscatter)r   r|   r_  rl  rW  )r   r|   r   r(  rU  rV  rD  s          r5   scatter_meta_implro  M  s;     hhj1Ky$s;56'3 r8   c                 R    [        XX#S5        U R                  U R                  5      $ NrR  ro  r   r   r   r|   r   r(  s       r5   meta_scatter_addrt  V  s!    dU3>>$**%%r8   c                      [        XX#S5        U $ rq  ro  rs  s       r5   meta_scatter_add_rw  \  s    dU3Kr8   c                     [        U[        R                  5      (       a  UOS n[        XX%U5        U R	                  U R
                  5      $ r2   )rf   rJ   r   ro  r   r   r   r|   r   src_or_valuerf  r(  s         r5   meta_scatterr{  b  s:     %\5<<@@,dCdV4>>$**%%r8   c                 f    [        U[        R                  5      (       a  UOS n[        XX%U5        U $ r2   )rf   rJ   r   ro  ry  s         r5   meta_scatter_r}  q  s,     %\5<<@@,dCdV4Kr8   queryr   rX  	dropout_p	is_causalreturn_debug_maskr  c           	      \   U R                  S5      nU R                  S5      nU R                  S5      n	U R                  S5      n
UR                  S5      nU R                  SS5      n[        R                  " U5      R                  SS5      n[        R                  " XxU	4[        R
                  U R                  S9nU(       aa  U
S:  a  SOSn[        R                  " X-  5      nUS::  a  SnOUS::  a  Sn[        R                  " XxU	U4U R                  U R                  S9nO*[        R                  " SU R                  U R                  S9n[        R                  R                  (       al  [        R                  R                  5       (       aI  [        R                  " S	[        R                  S
S9n[        R                  " S	[        R                  S
S9nOH[        R                  " S[        R                  S
S9n[        R                  " S	[        R                  S
S9nUUS S U	UUUU4	$ )Nr   r   r   r-   r  @         r=   rw   )r   r  rJ   r   r   rN   ry   r  ceilrR   versionhipr   r  r   rN  )r~  r   rX  r  r  r  r  r   	num_headsmax_seqlen_batch_qhead_dimmax_seqlen_batch_kquery_t	attention	logsumexpblocksize_cmax_seqlen_k
debug_maskseedoffsets                       r5   (meta__scaled_dot_product_flash_attentionr    s    AJ

1IAzz!}H!ooa#G  )33Aq9I	 23kk||I %]cyy!3!AB$L3&L[[$6E++<<

 [[%++ellK
 }}UZZ4466{{2UZZ?Ruzz&A{{Aell6BRu||FC 	
 
r8   	attn_biascompute_log_sumexpc	           	      
   U R                  S5      n	U R                  S5      n
U R                  S5      nUR                  S5      nUR                  S5      n[        R                  " XX4U R                  U R                  S9n[        R                  " XU4[        R
                  U R                  S9n[        R                  " S[        R                  SS9n[        R                  " S[        R                  SS9nUUS S UUUUS 4	$ Nr   r   r   r   r  r=   rw   r   rJ   r   rR   ry   rN   r   )r~  r   rX  r  r  r  r  r  r  r  r8  S_QS_KVD_Vr  
logsum_expr  r  s                     r5   (meta__scaled_dot_product_cudnn_attentionr    s     	

1A

1A
**Q-C88A;D
**R.C
++qS&ekk%,,
OC	
skk||J ;;rF;D[[5::f=F 	
 
r8   c           	      
   U R                  S5      nU R                  S5      n	U R                  S5      n
UR                  S5      nUR                  S5      n[        R                  " XX4U R                  U R                  S9n[        R                  " XU
4[        R
                  U R                  S9n[        R                  " S[        R                  SS9n[        R                  " S[        R                  SS9nUUS S U
UUUS 4	$ r  r  )r~  r   rX  r  r  r  r  r  r  r8  r  r  r  r  r  r  r  s                    r5   5meta__scaled_dot_product_fused_attention_overrideabler    s     	

1A

1A
**Q-C88A;D
**R.C
++qS&ekk%,,
OC	
skk||J ;;rF;D[[5::f=F 	
 
r8   r  r  	cum_seq_q	cum_seq_kmax_qmax_kphilox_seedphilox_offsetc                 P   [         R                  " UR                  SS5      5      R                  SS5      n[         R                  " UR                  SS5      5      R                  SS5      n[         R                  " UR                  SS5      5      R                  SS5      nUUU4$ r  )rJ   r   r  )r  r~  r   rX  r   r  r  r  r  r  r  r  r  r  r  grad_qgrad_kgrad_vs                     r5   'meta__scaled_dot_product_flash_backwardr    s    , eooa34>>q!DFcmmAq12<<QBFeooa34>>q!DF66!!r8   	attn_maskc                    U R                  S5      nU R                  S5      nU R                  S5      n	[        R                  " U 5      n
[        R                  " UU	U4[        R                  U R
                  S9R                  SS5      nU
U4$ )Nr   r   r   r  )r   rJ   r   r   rN   ry   r  )r~  r   rX  r  r  r  r  r   r  r  r  r  s               r5   0meta__scaled_dot_product_flash_attention_for_cpur  /  s     AJ

1IA  'I	

 kk|| i1o  	 r8   c
                    UR                  S5      n
UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      n[        R                  " XX4SUR                  UR                  S9n[        R                  " XX4SUR                  UR                  S9n[        R                  " XX4SUR                  UR                  S9nUUU4$ )Nr   r   r-   r   r   r   r   r-   r  )r   rJ   empty_permutedrR   ry   )r  r~  r   rX  r   r  r  r  r  r  r   r  r  len_qlen_kr  r  r  s                     r5   9meta__scaled_dot_product_flash_attention_for_cpu_backwardr  Q  s    & AJ

1Izz!}HJJqMEHHQKE!!	0kk||	F !!	0iizz	F !!	0kk||	F 66!!r8   c           	      R   U R                  SS5      n UR                  SS5      nUR                  SS5      nU R                  S5      nU R                  S5      n	U R                  S5      n
UR                  S5      n[        R                  " XXU R                  U R
                  S9n[        R                  R                  (       a0  [        R                  R                  5       (       a   U(       a  U	OSnO%U(       a  [        R                  " U	S-  5      S-  OSn[        R                  " XU4[        R                  U R
                  S9nUR                  SS5      n[        R                  " S[        R                  S	S9n[        R                  " S[        R                  S	S9nXUU4$ )
Nr   r   r   r  r   r  rN  r=   rw   )r  r   rJ   r   rR   ry   r  r  r   r  r  r  rN   r   )r~  r   rX  r  r  r  r  r  r  r  r  Kvr  logsumexp_dimr  r  r  s                    r5   ,meta__scaled_dot_product_efficient_attentionr    s?    OOAq!E
--1
COOAq!E

1A

1A

2I	BB
++aIU\\
RC}}UZZ4466	 0Q2D		!b&)B.!	
}%kk||J --1
C ;;rF;D[[5::f=FD&((r8   grad_input_maskc                    UR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      n[        R                  " XUU4SUR                  UR                  S9n[        R                  " XUU4SUR                  UR                  S9n[        R                  " XUU4SUR                  UR                  S9nS nUb  U
S   (       ax  UR                  S5      nUS-  S:X  a  UO
US-   US-  -
  n[        UR                  5       5      nUUS'   [        R                  " UUR                  UR                  S9nUS	S U24   nUUUU4$ )
Nr   r   r   r-   r  r  r   rX  .)r   rJ   r  rR   ry   r   r   )r  r~  r   rX  r  r   r  r  r  r  r  r  r  r   r  r  r  
head_dim_vr  r  r  r  	grad_biaslastDimlastDimAligned	new_sizess                             r5   +meta__scaled_dot_product_efficient_backwardr    su   ( AJ

1IJJqMEzz!}HAJHHQKE!!	x0kk||	F !!	x0iizz	F !!	z2kk||	F I!3..$$+bLA$57R<'TV,;V)*	&	"KKY__Y5E5E
	 c8G8m,	669,,r8   c                     [         R                  " U5      n[         R                  " U5      n[         R                  " U5      nUUU4$ r2   r}  )r  r~  r   rX  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                      r5   'meta__scaled_dot_product_cudnn_backwardr    sA    . e$Fc"Fe$F66!!r8   window_size_leftwindow_size_right	seqused_kalibi_slopesc                    Uc  U R                  S5      OUR                  5       S-
  nUc  U R                  S5      OUnUc  UR                  S5      OUnU R                  S5      nU R                  S5      n[        R                  " U 5      nUc2  [        R                  " UUU4[        R
                  U R                  S9nOAU R                  S5      n[        R                  " UU4[        R
                  U R                  S9nU	(       ac  US:  a  SOSn[        R                  " UU-  5      nUS::  a  SnOUS::  a  Sn[        R                  " UUUU4U R                  U R                  S9nO*[        R                  " SU R                  U R                  S9nS	u  nn[        R                  R                  (       al  [        R                  R                  5       (       aI  [        R                  " S
[        R                  SS9n[        R                  " S
[        R                  SS9nOH[        R                  " S[        R                  SS9n[        R                  " S
[        R                  SS9nUUUUU4$ )Nr   r   r  r   r  r  r  r  NNr=   rw   r   )r   r   rJ   r   r   rN   ry   r  r  rR   r  r  r   r  r   rN  )r~  r   rX  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  total_qr  r  r  r  r  s                               r5   meta__flash_attention_forwardr    s   4 #,"3A9JQ9NJ*3*;A(1(9!u

2Izz"~H   'IKK$67++<<
	 **Q-KK ELL
	 %]cyy!3k!AB$L3&L[[$6E++<<

 [[%++ellK
 LD&}}UZZ4466{{2UZZ?Ruzz&A{{Aell6BRu||FC r8   c                     [         R                  " U5      n[         R                  " U5      n[         R                  " U5      nUUU4$ r2   r}  )r  r~  r   rX  r   r  r  r  r  r  r  r  r  r  r  r  r  
grad_querygrad_key
grad_values                       r5   meta__flash_attention_backwardr  V  sA    0 !!%(J$H!!%(Jx++r8   cu_seqlens_qcu_seqlens_kmax_seqlen_qr  custom_mask_typecausal_diagonalseqlen_kwindow_sizec           	         U R                  S5      nU R                  S5      nUR                  S5      nU R                  S5      nUR                  S5      n[        R                  " UUUUU R                  U R                  S9nUb  UR                  S5      S-
  OUnUnUb  Uc   eUnUb  UOUnU
(       a  [
        R                  " US-  5      S-  OSn[        R                  " UUU4[        R                  U R                  S9n[        R                  " S[        R                  SS9n[        R                  " S[        R                  SS9nUUUUUU4$ )	Nr   r   r  r   r  rN  r=   rw   )	r   rJ   r   rR   ry   r  r  rN   r   )r~  r   rX  r:  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  logsumexp_batch_dimactual_max_seqlen_qactual_max_seqlen_kr  r  r  r  s                               r5   !meta__efficient_attention_forwardr  u  s9   , 	

1A

1AA

2I	BB
++aIrU\\
RC7C7O,++A.2VW'''**6*B,4F		%*+b0A  	i7kk||J ;;rF;D[[5::f=F
D&*=?RRRr8   bias_requires_gradnum_splits_keyshared_storage_dqdkdvc                    U(       a  [         R                  " UR                  S   UR                  S   :H  S 5        [         R                  " UR                  S   UR                  S   :H  S 5        [         R                  " / UR                  SS QSPUR                  S   PUR                  S   P7UR                  UR
                  S9nUR                  S	S5      nUR                  S	S5      nUR                  S	S
5      nOB[         R                  " U5      n[         R                  " U5      n[         R                  " U5      nUby  UR                  S5      nUS-  S:X  a  UO
US-   US-  -
  n[        UR                  5       5      nUUS'   [         R                  " UUR                  UR
                  S9nUSS U24   nO[         R                  " SUR
                  S9nUUUU4$ )Nr   c                      g)Nz,seqlen must match for `shared_storage_dqdkdvr=   r=   r8   r5   rY   4meta__efficient_attention_backward.<locals>.<lambda>  s    Br8   r-   c                      g)Nz3embedding dim must match for `shared_storage_dqdkdvr=   r=   r8   r5   rY   r    s    Ir8   r   r  r   r  rK  r   rX  .r=   r  )
rJ   r\   r   r   rR   ry   r  r   r   r   )r  r~  r   rX  r:  r  r  r  r  r  r  r  r  r  r  r  r  r  chunkr  r  r  r  r  r  r  s                             r5   "meta__efficient_attention_backwardr    s   2 KKNciil*B	
 	KKNciil*I	
 Eekk!BEEEKKOEU[[_E++<<

 \\"a(
<<A&\\"a(
%%e,
##C(%%e,
))B-$+bLA$57R<'TV,;V%	&	"KK	DKKP	c8G8m,	KK5<<8	xY66r8   scale_ascale_bscale_resultuse_fast_accumc                    ^ ^^^^^ S n[         R                  " T R                  5       S:H  =(       a    TR                  5       S:H  UU 4S j5        [         R                  " U" T R                  5      =(       a    U" TR                  5      UU 4S j5        [	        T 5      S:X  Ga  S n	S n
S n[         R                  " U	" T R                  5       5      =(       d    U" T 5      U 4S	 j5        [         R                  " U
" TR                  5       5      =(       d    U" T5      U4S
 j5        [         R                  " T R                  S5      S-  S:H  U 4S j5        [         R                  " TR                  S5      S-  S:H  =(       a    TR                  S5      S-  S:H  U4S j5        [         R                  " TR                  [         R                  :H  =(       a    TR                  [         R                  :H  S 5        T R                  u  mnTR                  S5      mTR                  5       S:X  a  TR                  5       S:X  a  O[         R                  " TR                  5       S:H  =(       a    TR                  5       S:H  UU4S j5        TR                  S5      T:X  a}  TR                  S5      S:X  ah  TR                  S5      S:X  aS  TR                  S5      T:X  a>  [         R                  " TR                  5       =(       a    TR                  5       S 5        O[         R                  " SUUUU4S j5        Ub  UOT R                  n[         R                  " T R                  S5      TR                  S5      UT R                  S9$ )Nc                     U [         R                  [         R                  [         R                  [         R                  4;   $ r2   )rJ   r\  float8_e5m2float8_e4m3fnuzfloat8_e5m2fnuzr   s    r5   is_fp8_type#meta_scaled_mm.<locals>.is_fp8_type  s8    !!!!	
 
 	
r8   r   c                  L   > STR                  5        ST R                  5        3$ )Nz%Inputs must be 2D but got self.dim()=z and mat2.dim()=r   rG  r   s   r5   rY    meta_scaled_mm.<locals>.<lambda>  s%    7
|CSTXT\T\T^S_`r8   c                  <   > STR                    ST R                    3$ )Nz8Expected both inputs to be fp8 types but got self.dtype=z and mat2.dtype=r   r  s   r5   rY   r    s     J4::,Vfgkgqgqfrsr8   r   c                 6    U S   U S   :  =(       a    U S   S:H  $ r  r=   r  s    r5   is_row_major$meta_scaled_mm.<locals>.is_row_major  s"    !9vay(;VAY!^;r8   c                 0    U S   S:H  =(       a    U S   S:  $ r  r=   r  s    r5   is_col_major$meta_scaled_mm.<locals>.is_col_major  s    !9>3fQi!m3r8   c                 `    U R                  S5      S:H  =(       d    U R                  S5      S:H  $ r  r   )	tensor_2ds    r5   has_zero_dim$meta_scaled_mm.<locals>.has_zero_dim	  s)    >>!$)CY^^A->!-CCr8   c                  *   > ST R                  5        3$ )Nz#self must be row_major, got stride r  r   s   r5   rY   r        9$++-Ir8   c                  *   > ST R                  5        3$ )Nz#mat2 must be col_major, got stride r  rG  s   r5   rY   r    r   r8   r   rX  r   c                  ,   > ST R                  S5       3$ )NzBExpected self.size(1) to be divisible by 16, but got self.size(1)=r   r   r   s   r5   rY   r    s    XY]YbYbcdYeXfgr8   c                  "   > ST R                    3$ )Nz>Expected both dimensions of mat2 to be divisble by 16 but got r?  r  s   r5   rY   r    s    TUYU_U_T`ar8   c                      g)Nz6Both scale_a and scale_b must be float (fp32) tensors.r=   r=   r8   r5   rY   r     s    Lr8   c                  P   > ST R                  5       < STR                  5       < 3$ )NzLFor non-tensorwise scaling, scale tensors must be 2D, but got scale_a.dim()=z and scale_b.dim()=r   )r  r  s   r5   rY   r  +  s*    gY`YdYdYfXhh|nunynyn{m}~r8   c                      g)Nz@Both scale_a and scale_b must be contiguous for rowwise scaling.r=   r=   r8   r5   rY   r  7  s    ^r8   Fc                     > ST  ST STR                  S5       STR                  S5       STR                  S5       STR                  S5       S3$ )	Nz}Invalid scaling configuration. For tensorwise scaling, both scales should be scalar. For rowwise scaling, scale_a should be (z, 1), scale_b should be (1, z). Got scale_a.size()=(r   ro   r   z) and scale_b.size()=(rp   r   )ra  r  r  r  s   r5   rY   r  =  sf    CCD#Eabcad e//6||A.?r',,q/AR S//6||A.?r',,q/ARRS	Ur8   r  )rJ   r\   r|   rR   r   r   r   rY  r   r   r]  r   ry   )r   rG  r  r  r:  r  r<  r  r  r  r  r  _k
_out_dtypera  r  s   ````          @@r5   meta_scaled_mmr    s   
 
LL
a+DHHJ!O` 
LLDJJ;K

$;s
 4F"	<	4	D 	'=<+=I	
 	'=<+=I	
 	IIaL2"g	
 	IIaL2"=tyy|b'8A'=a	
 	MMU]]*Mw}}/ML	
 

2IIaL==?aGMMOq$8 LL"9w{{}'9~ Q1$LLOq(LLOq(LLOq( ))+G0E0E0G^ 	 (3J;;tyy|TYYq\DKKXXr8   c           	      P    [        XX#USS9  U R                  U R                  5      $ NT)rV  rr  r   r|   r   r(  rf  rd  s         r5   meta_scatter_reduce_twor  J  s%     dVTJ>>$**%%r8   c           	          [        XX#USS9  U $ r  rv  r  s         r5   meta_scatter_reduce__twor  Q  s    dVTJKr8   c                |  ^  [         R                  " ST R                  5       s=:  =(       a    S:*  Os  U 4S j5        T R                  5       S:X  a.  [         R                  " U[         R                  T R
                  S9$ [         R                  " T R                  S5      U[         R                  T R
                  S9$ )Nr   r   c                  *   > ST R                  5        3$ )Nz@The probabilty distributions dimensions must be 1 or 2, but got r   rF  s   r5   rY   "meta_multinomial.<locals>.<lambda>\  s    RSXS\S\S^R_`r8   r   r  )rJ   r\   r|   r   r   ry   r   )r   num_samplesreplacementr  s   `   r5   meta_multinomialr  W  s     
LL	EIIK1` yy{a{{;ejjNN;;

1{%**U\\ r8   c                 $    SnU  H  nX-  nM	     U$ r-  r=   )vsrj  vs      r5   multiply_integersr  e  s    	A	 Hr8   c                 ^  ^ ^^^ [         R                  " [        T5      T:H  UU4S j5        TS-   m[         R                  " [        T 5      T:H  UU 4S j5        [         R                  " [        S T SS   5       5      =(       a    [        S T 5       5      U U4S j5        T S S u  p4X4/TQ7$ )Nc                  &   > ST  S[        T5       3$ )Nz%It is expected output_size equals to , but got size r  )num_spatial_dimsr  s   r5   rY   'upsample_common_check.<locals>.<lambda>o  s    78H7IY\]hYiXjkr8   r   c                  &   > ST  S[        T5       3$ )Nz$It is expected input_size equals to r  r  )expected_input_dimsrh  s   r5   rY   r   t  s    67J6K?[^_i[jZklr8   c              3   *   #    U  H	  oS :  v   M     g7fr  r=   rh   r  s     r5   rj   (upsample_common_check.<locals>.<genexpr>x  s     *>aE>r  c              3   *   #    U  H	  oS :  v   M     g7fr  r=   r$  s     r5   rj   r%  x  s     2N+Qq5+r  c                     > ST  ST 3$ )NzDInput and output sizes should be greater than 0, but got input size z and output size r=   )rh  r  s   r5   rY   r   y  s      \!2;-Ar8   )rJ   r\   r   r  )rh  r  r  r  channelsr"  s   ```  @r5   upsample_common_checkr)  l  s    	LLK,,k +Q.	LLJ..l
 
LL*:ab>**Ns2N+2N/N	A ""1~F+{++r8   c                 8  ^  [         R                  " T R                  5       S:g  =(       d    [        T R	                  5       SS  5      U 4S j5        [        T R	                  5       USS9nT R                  U5      R                  [        R                  " T 5      S9$ )Nr   r   c                  *   > ST R                  5        3$ )Nz>Non-empty 3D data tensor expected but got a tensor with sizes r   rF  s   r5   rY   $upsample_nearest1d.<locals>.<lambda>      PQVQ[Q[Q]P^_r8   r  r   
rJ   r\   r   r  r   r)  r   r%  rB   r  )r   r  scalesfull_output_sizes   `   r5   upsample_nearest1dr2         
LLA/

QR0@A_ -

kA ??+,//11%8 0  r8   c                   ^  [         R                  " T R                  5       S:g  =(       d    [        T R	                  5       SS  5      U 4S j5        [        T R	                  5       USS9nT R                  U5      n[        R                  " T 5      nT R                  u  px  nT R                  R                  S:X  a  US:  a  [         R                  nUR                  US9nU$ )	Nr   r   c                  *   > ST R                  5        3$ Nz>Non-empty 4D data tensor expected but got a tensor with sizes r   rF  s   r5   rY   $upsample_nearest2d.<locals>.<lambda>  r-  r8   r   r.  r   r  r   )rJ   r\   r   r  r   r)  r   rB   r  r   ry   rq   r   
contiguous)	r   r  scales_hscales_wr1  r   r   rE   
n_channelss	   `        r5   upsample_nearest2dr<    s     
LLA/

QR0@A_ -

kA __-.F //6M  ++A1a||F"zA~//];FMr8   r  rh  r9  r:  c                 X  ^ ^^ [        X!SS9m[        R                  " T R                  S:H  U 4S j5        [	        S5       H5  m[        R                  " T R                  T5      TT   :H  UU U4S j5        M7     T R                  U5      R                  [        R                  " T 5      S9$ )Nr   r.  r  c                  "   > ST R                    3$ NzFExpected grad_output to be a tensor of dimension 4 but got: dimension r	  r  s   r5   rY   -upsample_nearest2d_backward.<locals>.<lambda>      XYdYiYiXjkr8   c            
      D   > ST ST T    ST STR                  T5       3$ )NzCExpected grad_output to have the same shape as output; output.size() = z but got grad_output.size(r   r1  r  r   s   r5   rY   r@    s9      !s$'7':&;,QCtK4D4DQ4G3HJr8   r   )
r)  rJ   r\   r   r   r   r   r%  rB   r  )r  r  rh  r9  r:  r1  r   s   `    @@r5   upsample_nearest2d_backwardrE    s     -! 
LLAk 1XQ#3A#66	
    ,//11+> 0  r8   c                 8  ^  [         R                  " T R                  5       S:g  =(       d    [        T R	                  5       SS  5      U 4S j5        [        T R	                  5       USS9nT R                  U5      R                  [        R                  " T 5      S9$ )Nr   r   c                  *   > ST R                  5        3$ )Nz>Non-empty 5D data tensor expected but got a tensor with sizes r   rF  s   r5   rY   $upsample_nearest3d.<locals>.<lambda>  r-  r8   r-   r.  r   r/  )r   r  scales_dr9  r:  r1  s   `     r5   upsample_nearest3drJ    r3  r8   c                    [         R                  " U 5      [         R                  " U [         R                  S9pvUb  Ub  [        U[        5      (       d   e[        U[        5      (       d   eUR
                  nUR                  5       n	[        XH5      n[        XX5      nUR                  X5        UR                  X5        [        XdS9  [        XuS9  XE4$ Xg4$ )Nr   )r  r  )
rJ   r   r   rf   r   r   r   r    r   r"   )
r   stabler|   
descendingr   r   r  r   r   
out_strides
             r5   	meta_sortrO    s     D!5#3#3D#Lqg1&*----':.... GG	XXZ
"65#G791I2344Kr8   c           	      >  ^ ^^^^^^^ [         R                  " T R                  S:H  U 4S j5        [         R                  " T R                  TR                  :H  UU 4S j5        T R	                  S5      mTb  [         R                  " TR                  S:H  U4S j5        [         R                  " TR                  5       T:H  UU4S j5        [         R                  " TR                  TR                  :H  UU4S j5        [         R                  " TR                  S:H  U4S j5        T R	                  S	5      T-  T-  m[         R                  " TR                  5       T:H  UUUU U4S
 j5        [         R                  " [        U 4S jTTTT4 5       5      S 5        g )Nr   c                  "   > T R                    S3$ Nz != 2r	  )input_gatess   r5   rY   %rnn_cell_checkSizes.<locals>.<lambda>      ;3C3C2DE0Jr8   c                  :   > TR                    ST R                    3$ N != r?  )hidden_gatesrS  s   r5   rY   rT    s    ;$$%T,*<*<)=>r8   r   c                  "   > T R                    S3$ )Nz != 1r	  )
input_biass   r5   rY   rT    s    joo5Fe3Lr8   c                  .   > TR                  5        ST  3$ rW  r  )
gates_sizer[  s   r5   rY   rT    s    z'')*$zl;r8   c                  :   > TR                    ST R                    3$ rW  r?  )hidden_biasr[  s   r5   rY   rT    s    z''([->->,?@r8   c                  "   > T R                    S3$ rR  r	  )prev_hiddens   r5   rY   rT    rU  r8   r   c            
      `   > TR                  5        STR                  S5       ST ST ST  S3
$ )NrX  r   z * z // z (aka rp   )r   r   )expected_prev_hidden_numelfactorr]  rS  ra  s   r5   rY   rT    s@    ;$$&'tK,<,<Q,?+@J<tTZS[[ab|a}}~r8   c              3   V   >#    U  H  nUR                   TR                   :H  v   M      g 7fr2   r  )rh   rG   rS  s     r5   rj   &rnn_cell_checkSizes.<locals>.<genexpr>  s'      
I HH***Is   &)c                      g)Nz%expected all inputs to be same devicer=   r=   r8   r5   rY   rT    s    7r8   )rJ   r\   r   r   r   r   r  )rS  rY  r[  r_  rd  ra  rc  r]  s   ``````@@r5   rnn_cell_checkSizesrh    sC    
LL!!Q&(JK	LL\///> !!!$JZ__)+LM*,;	
 	 1 11@	
 
LL!!Q&(JK!,!1!1!!4z!AV!K	LL99 
LL 
"J[I
 	
 	8r8   c                     [        XX4SU5        [        R                  " U [        R                  S9n[        R                  " U[        R                  S9n[        R                  " U[        R                  S9nXgU4$ )Nr  r   )rh  rJ   r   r   )rS  rY  cxr[  r_  	workspacehycys           r5   _thnn_fused_lstm_cell_metarn     se     :ArR  E<S<STI			"E,C,C	DB			"E,C,C	DBIr8   c                    [        U5      S:g  nU(       a   [        U5      nUS   nU R                  S   nOLU
(       a  U R                  S   OU R                  S   nU
(       a  U R                  S   OU R                  S   nSnU(       a  SOSnUS:w  a  UOUnU(       a  UUU-  /nOU
(       a  UUUU-  /OUUUU-  /nU R                  U5      nU	U-  UU/nUc   [        R                  " SU R
                  S9nOUR                  U5      nUR                  U	U-  UU/5      nU(       a  SOSnU R                  U[        R                  S9nUUUUU4$ )Nr   r   r   r   r  r   )r   r   r   rJ   r   ry   r  )r   r8  weight_stride0
weight_bufhxrj  rf  hidden_size	proj_size
num_layersbatch_firstdropouttrainbidirectionalbatch_sizesdropout_stateis_input_packed
seq_length
mini_batchbatch_sizes_sumnum_directionsout_sizer   r   
cell_shaperm  rl  reserve_shapereserves                                r5   
_cudnn_rnnr  /  sS   & +&!+O%
 ^
++a.'2U[[^A
'2U[[^A
'QQN%NyH$h&?@	  X%>?j(^*CD 	
 __Y'F~-z;GJ	z[[5<<0\\*%	zN2JI	JB AAMoom5;;o?G2r7J..r8   c                 >   U(       a  U R                   S   OU R                   S   nU(       a  U R                   S   OU R                   S   nU
nU(       a  UUU/OUUU/nU R                  U5      nUc   [        R                  " SU R                  S9nOUR                  UR                   5      nUc   [        R                  " SU R                  S9nOUR                  UR                   5      n[        R                  " SU R                  [        R
                  S9nUUUU4$ )Nr   r   r  r   )r   r   rJ   r   ry   r  )r   w0w1w2w3hx_cx_r   rz  rf  rs  ru  
has_biasesry  rv  rx  r}  r~  output_chanelsr   r   rl  rm  rk  s                           r5   mkldnn_rnn_layerr  g  s    & $/QEKKNJ#.QEKKNJ N  
Z0*n5 
 __Y'F
{[[5<<0]]399%
{[[5<<0]]399%Aell%++FI2r9$$r8   c                    ^^ U R                   S:X  a,  [        R                  " TS:H  =(       d    TS:H  UU4S j5        g [        R                  " U R                  T5      S:g  UU4S j5        g )Nr   r   c                     > T ST  3$ )Nz4: Expected reduction dim -1 or 0 for scalar but got r=   r|   r  s   r5   rY   'zero_numel_check_dims.<locals>.<lambda>  s    wiSTWSXYr8   c                     > T ST  S3$ )Nz: Expected reduction dim z to have non-zero size.r=   r  s   r5   rY   r    s    wi8=TUr8   )r   rJ   r   r   )r   r|   r  s    ``r5   zero_numel_check_dimsr    sR    yyA~1H!r	Y	

 	IIcNaU	
r8   c                    ^  Ub&  [        X!R                  5       5      n[        XT 5        g [        R                  " UR                  5       S:g  U 4S j5        g )Nr   c                     > T  S3$ )Nz@: Expected reduction dim to be specified for input.numel() == 0.r=   r3  s   r5   rY   %check_argmax_argmin.<locals>.<lambda>  s    tf\]r8   )r   r|   r  rJ   r\   r   )r  r   r|   s   `  r5   check_argmax_argminr    s?    
S((*-d.JJLA]	
r8   c                     [        SX5        [        R                  " U R                  Ub  U4OS 5      n[	        XU5      nU R                  U[        R                  S9$ )Nargmaxr   )r  rB   r  r   r  r   rJ   r   )r   r|   r  r  r   s        r5   argmax_argmin_metar    sM    $,

coSF4PD$T9E>>%u{{>33r8   c                 v    U[         R                  :X  a  [         R                  n[         R                  " SXX4S9$ )Nr=   r  )rJ   jaggedr  r   )r  rR   rx   ry   rz   s        r5   scalar_tensorr    s1    
 ;;
%v r8   c                    [        X R                  5       SS9nU R                  5       S:X  a  SOU R                  U5      n[        R                  " U5        [        R
                  " X:*  S 5        [        U R                  5      n[        U5      S:  a  XU'   U R                  U5      U R                  U[        R                  S94$ )NTr3  r   r   c                      g)Nzk not in range for dimensionr=   r=   r8   r5   rY   topk_meta.<locals>.<lambda>  s    )Gr8   r   )r   r|   r   rJ   r  r\   r   r   r   r   r   )r   r`  r|   largestsorted	sliceSizetopKSizes          r5   	topk_metar    s     hhjd
;CXXZ1_$))C.I		LL!GHDJJH
8}q>>(#T^^HEKK^%PPPr8   c                     Uc
  Uc   S5       eUR                  5       nU R                  5       n	[        R                  " UU	R                  U	R                  U	R
                  S9$ )Nz;segment_reduce(): Either lengths or offsets must be defined)rR   ry   rx   )r8  rJ   r   rR   ry   rx   )
rV  r   rv  rf  rq  rr  rs  ru  data_contiggrad_contigs
             r5   meta__segment_reduce_backwardr    sj    
 '"5 E5 //#K//#K!!!!	 r8   c                   ^ [        TU R                  5       SS9mU R                  5       S:  a  U R                  T5      OSn[        R                  " US:  =(       a    X:*  U4S j5        [        U R                  S T U R                  TS-   S  -   5      nU(       a&  U R                  5       S:  a  UR                  TS5        U R                  U5      U R                  U[        R                  S94$ )NTr  r   r   c                     > ST  3$ )Nz9kthvalue(): selected number k out of range for dimension r=   r   s   r5   rY   kthvalue_meta.<locals>.<lambda>  s    KC5Qr8   r   )
r   r|   r   rJ   r\   r   r   r.  r   r   )r   r`  r|   r  dimSizer   s     `   r5   kthvalue_metar    s     dhhjd
;C $
QdiinAG	LL	Q1<Q
 DS!DJJsQwy$99:E488:>S!>>% $..ekk."JJJr8   c                    U b  U OUn[         R                  " UR                  5       S:H  S 5        UR                  5       nU b)  [         R                  " U R                  5       U:H  S 5        Ub)  [         R                  " UR                  5       U:H  S 5        [         R                  " UR                  5       U:H  S 5        [         R                  " UR                  5       U:H  S 5        [         R                  " UR                  5       S:H  S 5        [         R                  " UR	                  5       US   US	   -  S
-  :H  S 5        g )Nr   c                      gN r=   r=   r8   r5   rY   (checkLSTMBackwardSizes.<locals>.<lambda>  s    "r8   c                      gr  r=   r=   r8   r5   rY   r        r8   c                      gr  r=   r=   r8   r5   rY   r    r  r8   c                      gr  r=   r=   r8   r5   rY   r        r8   c                      gr  r=   r=   r8   r5   rY   r    r  r8   c                      gr  r=   r=   r8   r5   rY   r    s    rr8   r   r   r  c                      gr  r=   r=   r8   r5   rY   r    s    Rr8   )rJ   r\   r|   r   r   )grad_hygrad_cyrj  rm  rk  defined_gradexp_sizes          r5   checkLSTMBackwardSizesr    s    %17wL	LL!!#q(*5  "HW\\^x/<W\\^x/<	LLh&
3	LLh&
3	LLA%z2	LL"hqkHQK&?!&CCZPr8   c                     U c  Uc  g[        XX#U5        [        R                  " U[        S9n[        R                  " U[        S9nU(       a  UR	                  SSS9OS nXgU4$ )NNNNr   r   F)r  )r  rJ   r   legacy_contiguous_memory_formatr  )	r  r  rj  rm  rk  has_bias
grad_gatesgrad_cxr  s	            r5   #_thnn_fused_lstm_cell_backward_implr    sf    7?7RY?!!!@J r1PQG4<
q%0$I	))r8   c                 2   S nS nS nUS   (       a  UR                  U R                  5       5      nUS   (       d
  US   (       aQ  UR                  UR                  S5      U R                  S5      45      nUR                  UR                  S5      5      nXEU4$ )Nr   r   r   r   r  )r  r  r	  r  r  grad_weightr  s          r5   linear_backwardr    s    JKI1~!++FKKM:
1~Q",,l.?.?.CV[[QS_-UV **<+<+<R+@A	Y//r8   c                   ^ ^ [        T R                  5      S:  a  T R                  S   X-  -  S:X  d   ST R                   SU 35       eS mUU 4S jnT R                  S   X-  -  nT R                  S   U-  nT R                  S	   U-  n/ T R                  S S QUPUPUP7nT R                  U5      nUR                  U" 5       S
9nU$ )Nr   rK  r   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 b    [         R                  R                  U 5      [         R                  :H  $ r2   r  r  s    r5   r  ,meta_pixel_shuffle.<locals>.is_channels_last"  s$    ""88=ATATTTr8   c                  F  > T " T5      (       a/  [        T5      S:X  a  [        R                  $ [        R                  $ TR	                  [        R                  S9(       a  [        R                  $ TR	                  [        R
                  S9(       a  [        R
                  $ g r  )r   rJ   r   r  r]  r  )r  r   s   r5   r  .meta_pixel_shuffle.<locals>.pick_memory_format%  s    D!!4 F*...***e.E.EF***e.C.CD((( Er8   r  r   r   )r   r   r   r%  )	r   upscale_factorr  r  HrWrr   r   r  s	   `       @r5   meta_pixel_shuffler    s     	DJJ!

2.2Q RVW W 2$**=TUcTdeW
U	) 	

2>:;A	B.	(B	B.	(B-$**Sb/-1-b-"-I
..
#C
&&13&
4CJr8   c                 X   U R                  U R                  5      nUR                  UR                  5      nUR                  UR                  5      nUR                  UR                  5      nUR                  UR                  5      nUR                  UR                  5      nUUUUUUU4$ r2   r  )r   weight0weight1weight2weight3r  cx_tmpr   hy_cy_grad_output_r_optgrad_hy_r_optgrad_cy_r_optr   rf  rs  ru  r  rx  ry  rz  rv  rk  diff_xdiff_hxdiff_cxdiff_w1diff_w2diff_bs                                r5   mkldnn_rnn_layer_backwardr  :  s    4 __U[[)FmmCII&Gv||,G.G.Gw}}-F7GVVWgEEr8   )	out_int32r   c                    [         R                  " U U(       a  [         R                  O[         R                  [         R                  S9$ )NrR   r   )rJ   r   r@  r   r   )r   
boundariesr  r   s       r5   meta_bucketizer  ]  s2     &ekkEKK-- r8   c                 D  ^ ^^^^ Sm[        T 5      S:X  a)  [        R                  " T R                  5       U 4S j5        [        R                  " [	        T[
        5      UU4S j5        [        R                  " TS:  UU4S j5        [        R                  " [	        T[        5      UU4S j5        [        R                  " [	        T[        5      UU4S j5        [        R                  " TT:  S	 5        [        R                  " TT R                  T R                  S
9$ )Nzhistc()r  c                  $   > ST R                    S3$ )Nz%"histogram_cpu" not implemented for 'rC  r   rF  s   r5   rY   meta_histc.<locals>.<lambda>n  s    =ekk]!Lr8   c                  $   > T S[        T 5       3$ )Nz#: argument 'bins' must be int, not r  binsr  s   r5   rY   r  r  s    7)>tDzlKr8   r   c                     > T ST  3$ )Nz: bins must be > 0, but got r=   r  s   r5   rY   r  t  s    gY.J4&#Qr8   c                  $   > T  S[        T5       3$ )Nz%: argument 'min' must be Number, not r  )r  r   s   r5   rY   r  w      7)@cLr8   c                  $   > T  S[        T5       3$ )Nz%: argument 'max' must be Number, not r  )r  r  s   r5   rY   r  {  r  r8   c                      g)Nz&{fn_name}: max must be larger than minr=   r=   r8   r5   rY   r  }  s    %Mr8   r   )
r   rJ   r\   r   rf   r   r   r   ry   rR   )r   r  r   r  r  s   ````@r5   
meta_histcr  g  s     G5U"##%L	
 
LL4!K 
LLQR	LL3L 
LL3L 
LLMN;;tELLDDr8   c                 F  ^  [        T R                  5       USS9n[        R                  " T R	                  5       S:g  =(       d#    [        S T R                  5       SS   5       5      U 4S j5        T R                  U5      R                  [        R                  " T 5      S9$ )Nr   r.  r   c              3   *   #    U  H	  oS :  v   M     g7fr  r=   )rh   r   s     r5   rj   ,meta_upsample_bimode2d_aa.<locals>.<genexpr>  s     !H7Gt(7Gr  r   c                  *   > ST R                  5        3$ r6  r   rF  s   r5   rY   +meta_upsample_bimode2d_aa.<locals>.<lambda>  r-  r8   r   )
r)  r   rJ   r\   r   r  r   r%  rB   r  )r   r  r	  r9  r:  r1  s   `     r5   meta_upsample_bimode2d_aar    s     -

kA 
LLHc!Huzz|AB7G!HH_ ??+,//11%8 0  r8   c                 T  ^ ^^ [        X!SS9m[        R                  " T R                  S:H  U 4S j5        [	        S5       H3  m[        R                  " T R
                  T   TT   :H  UU U4S j5        M5     T R                  U5      R                  [        R                  " T 5      S9$ )Nr   r.  r  c                  "   > ST R                    3$ r?  r	  r  s   r5   rY   4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>  rA  r8   c            
      D   > ST ST T    ST STR                  T5       3$ )NzD
Expected grad_output to have the same shape as output; output.size(rC  z
but got grad_output_size(r   rD  s   r5   rY   r	    s>     DDE3dK[\]K^J_ `D!1!1!!4 59r8   r   )
r)  rJ   r\   r   r   r   r   r%  rB   r  )r  r  rh  r	  r9  r:  r1  r   s   `     @@r5   "meta_upsample_bimode2d_aa_backwardr	    s     -! 
LLAk 1Xa $4Q$779	
    ,//11+> 0  r8   c                 X   [         R                  " UR                  5       S:H  S 5        [         R                  " UR                  5       S:H  S 5        [         R                  " UR                  R                  S 5        [         R                  " UR                  R                  S 5        g )Nr   c                      g)Nz%found_inf must be a 1-element tensor.r=   r=   r8   r5   rY   <_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>      (Or8   c                      g)Nz%inv_scale must be a 1-element tensor.r=   r=   r8   r5   rY   r	    r	  r8   c                      g)Nz!found_inf must be a float tensor.r=   r=   r8   r5   rY   r	        3r8   c                      g)Nz!inv_scale must be a float tensor.r=   r=   r8   r5   rY   r	    r	  r8   )rJ   r\   r   rR   r   )r   r  	inv_scales      r5   *_amp_foreach_non_finite_check_and_unscale_r	    s|    	LLQ O 
LLQ O 
LL))3 
LL))3r8   c                 V    [        U R                  5       5      nU R                  U5      $ r2   )r   r   r   )r   nanposinfneginfro  s        r5   
nan_to_numr	    s#     tyy{#K>>+&&r8   c                    U R                   [        R                  [        R                  [        R                  [        R
                  1;  d   SU R                    S35       eU R                  n[        X5      n[        X#5      nX:X  a  U $ [        U R                  5       5      n[        U R                  5       5      nXR   XQ   sXQ'   XR'   XB   XA   sXA'   XB'   U R                  XE5        U $ )Nz>torch.transpose_: in-place transposition is not supported for z layout)rx   rJ   r  
sparse_cscr  
sparse_bscr   r   r   r   r   r   )r   dim0rn  ndimsr   r   s         r5   r  r    s    ;;	   IU\]  IIE$&D$&D|		D$++- F!'v|FL&,!ZDJ
T"Kr8   c                 "   U R                   nU R                  (       a;  U R                  5       nU R                  5       nUS::  a  US:X  d   SU SU S35       eOU R	                  5       S::  d   SU S35       e[        U SUS:  a  S5      $ S5      $ )	Nr   r   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is r!  r   )r   r  r  r  r|   r  )r   r	  r  r  s       r5   t_r	    s    IIE~~__&
NN$	Q9> 	
!l,yk9JL	
1>
 xxzQ 	
DUG1M	
 dAEAIq55155r8   )r  r   sidesorterc                  ^ ^^ [         R                  " [        T R                  5      S:*  =(       d    T R                  S S TR                  S S :H  UU 4S j5        [         R                  " TS L =(       d    T R                  TR                  :H  U U4S j5        [         R                  " US:g  =(       d    U(       + S5        U(       a  [         R                  O[         R
                  n[        T[         R                  5      (       a$  [         R                  " TU[         R                  S9$ [         R                  " SUT R                  S	9$ )
Nr   r   c                  `   > S[        TR                  5       S[        T R                  5       3$ )Nztorch.searchsorted(): boundaries tensor should be 1 dimension or the first N-1 dimensions of boundaries tensor and input value tensor must match, but we got boundaries tensor z and input value tensor r   r   )r   sorted_sequences   r5   rY   #meta_searchsorted.<locals>.<lambda>  s3    3378M8M3N2O P""&tzz"2!35r8   c                  n   > S[        T R                  5       STb  [        TR                  5       3$ /  3$ )Nz[torch.searchsorted(): boundary and sorter must have the same size, but got boundary tensor z and got sorter tensor r	  )r 	  r	  s   r5   rY   r!	    sB    ##'(=(=#>"??V%+%7tFLL!@B=?@Br8   r   zetorch.searchsorted(): side and right can't be set to opposites, got side of left while right was Truer  r=   r  )rJ   r\   r   r   r@  r   rf   r   r   r   r   ry   )r 	  r   r  r   r	  r	  rR   s   ``   ` r5   meta_searchsortedr#	    s     
LLO!!"a' 	9  "%CR8	
	 
LL$?///6<<?	
 
LL#e)	$ %EKK%++E$%%U-D-D
 	
 {{2U?3I3IJJr8   c                    ^  [         R                  " T [         R                  [         R                  [         R                  4;  U 4S j5        g )Nc                     > ST  3$ )Nz/Unsupported input type encountered for isin(): r=   r   s   r5   rY   3_check_for_unsupported_isin_dtype.<locals>.<lambda>3  s    A%Ir8   )rJ   r\   r  
complex128	complex64r   s   `r5   !_check_for_unsupported_isin_dtyper)	  0  s/    	LLejj%"2"2EOODDIr8   c                 t    U	(       a  [         R                  U UUUUUUUU
U5
      $ [        U UUUUUUUU
U5
      $ r2   )r*   _embedding_bag_sparse_backward!meta_embedding_bag_dense_backward)rV  r   rr  r  r  maximum_indicesnum_weightsr  rf  r  r  r  s               r5   meta_embedding_bag_backwardr/	  7  se     22
 	
 1
 	
r8   c
                 X  ^  [         R                  " T R                  [         R                  [         R                  [         R
                  [         R                  4;   U 4S j5        U[        :X  a  [         R                  " US L5        T R                  UT R                  S5      45      n
U
$ )Nc                  "   > ST R                    3$ )Nz$Unsupported input type encountered: r   )rV  s   r5   rY   3meta_embedding_bag_dense_backward.<locals>.<lambda>q  s    6tzzlCr8   r   )
rJ   r\   rR   rZ  r[  rY  float64r  r   r   )rV  r   r  r  r-	  r.	  r  rf  r  r  index_grad_weights   `          r5   r,	  r,	  b  sv     
LL

u}}ennemmU]]SSC x_D01TYYq\'BCr8   c                    U R                  S5      n[        R                  " U[        :H  S5        [        R                  " U R	                  5       S:H  5        [        R                  " UR	                  5       S:H  5        UR                  S5      n[        R                  " UR	                  5       S:H  5        [        R                  " UR                  S5      U:H  5        U R                  U45      n	U	$ )Nr   zHembedding_bag_backward: per_sample_weights only supported for mode='sum'r   r   )r   rJ   r\   r  r|   r   )
rV  r8  r   rr  r  rf  r  embedding_featuresr  r   s
             r5   .meta_embedding_bag_per_sample_weights_backwardr7	  y  s     1	LLR 
LLq!	LL!#$,,q/K	LL"#	LLQ#556^^[N+FMr8   )assume_uniqueinvertc                   [         R                  " [        U [        5      =(       d    [        U[        5      S 5        [        U [        5      (       d  [         R                  " XR
                  S9n [        U[        5      (       d  [         R                  " XR
                  S9n[        U R                  5        [        UR                  5        [         R                  " U [         R                  S9$ )Nc                      g)Nz<At least one of elements and test_elements must be a Tensor.r=   r=   r8   r5   rY   meta_isin.<locals>.<lambda>  r  r8   r  r   )
rJ   r\   rf   r   r/  ry   r)	  rR   r   r  )elementstest_elementsr8	  r9	  s       r5   	meta_isinr?	    s     
LL8V$I
=&(IN h''<<1E1EFmV,,]??K%hnn5%m&9&9:HEJJ77r8   r  c                     [         R                  " U S:  S 5        [        U[        R                  S9u  p#[         R
                  " XS9$ )Nr   c                      g)Nz,polygamma(n, x) does not support negative n.r=   r=   r8   r5   rY    meta_polygamma.<locals>.<lambda>  s    !Or8   r  r   )rJ   r\   r   r   r  r   )r  r   rE   rF   s       r5   meta_polygammarC	    sB     
LLaOP(;HHOA D55r8   c                     [        S5      e)Nz.Tensor.item() cannot be called on meta tensors)r$  r   s    r5   meta_local_scalar_denserE	    s    
G
HHr8   c                 .    [         R                  " U 5      $ r2   r}  r   s    r5   silurG	    r  r8   c                 ^    [        U [        R                  S9u  p[        R                  " XS9$ r  )r   r   r  rJ   r   )r   rE   rF   s      r5   sigmoidrI	    s/     );HHOA D55r8   rG   half_to_floatc                    U(       a   U R                   [        R                  :X  d   e[        R                  " U [        R
                  R                  S9u  p4U(       d  UOUn[        R                  " X[        R                  S9nU$ )Nr  r  )	rR   rJ   rL   rB   r   r   rC   r   r   )rG   r|   rJ	  computation_dtyperF   r  s         r5   softmaxrM	    sk     ww%**$$$&+&>&>	uDDLL'# (5<:KL


1@W@W
XCJr8   r  r  r  c                    U R                  5       S:X  d   S5       eU R                  nUR                  nUR                  S:X  a  US   4nO%UR                  S:X  a  US   US   4nO
/ UQUS   P7nU R                  nU R	                  XxS9$ )Nr   z'weight' must be 2-Dr   r   r   )r|   r   r   rR   r   )	r8  r   r  r  r  weight_shapeindices_shaper   r<  s	            r5   	embeddingrQ	    s     ::<1444<<LMMM||q&21o%7			"1%|A7	5m5\!_5	II77r8   max_lengthspadding_valuec                     [        U5      S:X  d   e[        U5      S:X  d   eUS   R                  S   S-
  nUS   nXE/U R                  SS  Q7nU R                  U5      $ r  )r   r   r   )r   rr  rR	  rS	  r  r  rb  s          r5   $meta__jagged_to_padded_dense_forwardrU	    st     w<1{q   
aAAA,6<<+,LL))r8   c                 B    [        U 5      [        5       S 5       5       nU$ )Nc                 2    [        U [        R                  S9$ r)  rH   r   r  r_  s    r5   _f)_create_unary_float_meta_func.<locals>._f  s      =JJ
 	
r8   r>   r#   funcrY	  s     r5   _create_unary_float_meta_funcr^	    *    4]
  

 Ir8   c                 B    [        U 5      [        5       S 5       5       nU$ )Nc                 2    [        X[        R                  S9$ r)  rX	  )rG   rz  s     r5   rY	  *_create_binary_float_meta_func.<locals>._f  s      !@!M!M
 	
r8   r[	  r\	  s     r5   _create_binary_float_meta_funcrc	    r_	  r8   c                    ^  [        T 5      U 4S j5       nT R                   S3nX!l        [        [        [        U5      5      " U5      nU$ )Nc                 `   > T" U /UQ70 UD6n[        U R                  UR                  5        U $ r2   rW  )r   rD   r,  r   r4   s       r5   _fn#_register_inplace_meta.<locals>._fn*  s.    '''

CII6r8   rE   )r   rr   r>   getattrr*   )r4   rf	  inplace_names   `  r5   _register_inplace_metarj	  )  sK    
2Y 
 kk]!$LL
l3
4S
9CJr8   c                 z  ^ ^^ [         R                  " T R                  TR                  :H  UU 4S j5        T T/n[        T[        5      (       aT  TR
                  S:w  a3  [         R                  " T R                  TR                  :H  U U4S j5        UR                  T5        [        US[        R                  06$ )Nc                  <   > STR                    ST R                    3$ )Nr  z for `end`, but got dtype r   )rs   rt   s   r5   rY   lerp.<locals>.<lambda><  s    /%++.HTr8   r   c                  <   > ST R                    STR                    3$ )Nr  z for `weight`, but got dtype r   )rt   r8  s   r5   rY   rm	  C  s    /%++6STZT`T`Sabr8   r?   )
rJ   r\   rR   rf   r   r   r   rH   r   rC   )rt   rs   r8  rD   s   ``` r5   lerpro	  7  s     
LLsyy T 3<D&*%%;;!LLv||+b 	F	=EE r8   )rX  c                4    [        XU[        R                  S9$ r)  r+  r   tensor1tensor2rX  s       r5   addcmulrt	  K  s     0O0W0W r8   c                    [         R                  " [        R                  " UR                  5      =(       a     [        R                  " UR                  5      (       + S 5        [        XU[        R                  S9$ )Nc                      g)N)zFInteger division with addcdiv is no longer supported, and in a future zErelease addcdiv will perform a true division of tensor1 and tensor2. z4The historic addcdiv behavior can be implemented as zA(input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) zfor integer inputs and as z6(input + value * tensor1 / tensor2) for float inputs. z?The future addcdiv behavior is just the latter implementation: z4(input + value * tensor1 / tensor2), for all dtypes.r=   r=   r8   r5   rY   addcdiv.<locals>.<lambda>[  s     	
r8   r*  )rJ   r\   rB   rD  rR   rH   r   rC   rq	  s       r5   addcdivrx	  S  s`     
LL""7==1 6&&w}}5	
		
  0O0W0W r8   c                      0 n S H"  n[         U   nU H  nX0;  d  M
  X#   X'   M     M$     U R                  5        GH  u  pE[        U[        R                  R
                  5      (       a  M1  [        U[        5      (       d   eUR                  [        R                  R                  R                  5      " U5        [        R                  R                  UR                  5       S5      (       a  U[         S   ;   a  [        U S35      eM  UR                  (       a  M  UR                  5       S;   a  M  SUR                  5       ;   a  [        R!                  XE5        GM&  SUR                  5       ;   a  ["        R!                  XE5        GMR  SUR                  5       ;   a  [$        R!                  XE5        GM~  S	UR                  5       ;   a  [&        R!                  XE5        GM  [(        R!                  XE5        GM     g )
N)rw   post_autogradpre_autogradCompositeImplicitAutogradrw   z is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   aten::cloneaten::copy_aten::rot90aten::_to_copyaten::empty_stridedaten::constant_pad_ndaten::as_strided_scatterzmkldnn::zmkl::zonednn::zquantized::)r   itemsrf   rJ   _opsHigherOrderOperatorr   py_impl_CDispatchKeyr,   %_dispatch_has_kernel_for_dispatch_keyr  r$  is_view2_meta_lib_dont_use_me_use_register_meta_for_mkldnnimpl/_meta_lib_dont_use_me_use_register_meta_for_mkl2_meta_lib_dont_use_me_use_register_meta_for_onednn5_meta_lib_dont_use_me_use_register_meta_for_quantized'_meta_lib_dont_use_me_use_register_meta)activate_meta_tablerq   registryopoop_overloadr4   s         r5   activate_metar	  w  s    :-d3C-+3=#(  : /446
 k5::#A#ABB+z2222EHH00556r:8899 ;
 
 8@@""m $; ;  A    	 [--//BGGXK,,..?DD[U{//11BGGX+"2"2"44EJJ 8<<[Mm 7r8   )Fr  r2   )NNNFr   r   Fr  )Tr  )r  )r  T)FF)TT)rh  )FTN)TFF)TF)r   )g      ?N)r.   str)r=   r  r  F)r=   r  FTN)Fr   FNFr   )NF)r   F)g      ?gUUUUUU?FN)NNNNN)r   NNr   )NNF)        FFN)Nr	  FFN)r	  FNN)r	  FN)FN)FNNNN)NNNF)Nr   FNN)NNNN)r   TT)NNr   N)d   r   r   )r   )r   FF)r	  (  r  collections.abcr   enumr   	functoolsr   typingr   r   r   r	   typing_extensionsr
   rJ   torch._prims_commonr  rB   r   r   r   torch._decompr   r   r   r   
torch._opsr   torch._primsr   r   r   r   r   r   r   r   r   r   r   r   torch._prims_common.wrappersr   r    r!   r"   r#   r  r$   r%   torch.fx.experimentalr&   r  torch.utilsr'   r9   r(   r)   opsr*   libraryLibraryr	  r   r  r  r  r>   rH   rT   r^   linspacelogspacer  r   taker'  r   r   r   r   cummaxcumminr   r   r   r   r   r  r   _fft_c2cr   r   r   _fft_r2cr   randpermgenerator_outr  r   r
  randintr  r  low_outr  randr  _fft_c2rr  r  r+  r3  
unsqueeze_r7  _sparse_semi_structured_linearr	  rR   rD  _sparse_semi_structured_mmrJ  _sparse_semi_structured_addmmrN  _cslt_sparse_mmrc  index_reducerj  index_reduce_rl  index_selectrp  segment_reducer~  r  	unary_outr  r|   r  r   r  r  r  r  r  _assert_asyncr  msgr  _printr  _make_dep_tokenr  r  _functional_sym_constrain_ranger  r  (_functional_sym_constrain_range_for_sizer  _functional_assert_asyncr  r   r  r   r  r  r  r  _linalg_eighr  r  _linalg_eigvalslinalg_eigvalsr  
linalg_eigr  r  r  r  r  r  r  r/  linalg_inv_exr2  linalg_ldl_factor_exr[   r8  linalg_ldl_solverH  	linalg_lurO  linalg_lu_factor_exrT  linalg_lu_solver^  	lu_unpackre  ro  	linalg_qrrv  rz  rw  _linalg_svdr  rE  r  r  r  linalg_solve_triangularr  r  r  _linalg_detr  r  r  r  reflection_pad1dr  replication_pad1dr  r  reflection_pad1d_backwardr  replication_pad1d_backwardr  r'  reflection_pad2dr*  replication_pad2dr,  reflection_pad2d_backwardr  replication_pad2d_backwardr4  rC  reflection_pad3drF  replication_pad3drH  reflection_pad3d_backwardreplication_pad3d_backwardrP  _pdist_forwardrN   rU  _pdist_backwardr\  baddbmmrr  	bernoulliru  
bernoulli_rx  rQ  r{  poissonr~  _fused_moving_avg_obs_fq_helperr  mmr  r  r   r  r  miopen_batch_normr  convolutionr  r	  _has_mkldnnr	  r  _convolution_pointwiser  _linear_pointwiser  has_mklr	  r  _mkl_linearr  r	  r  qconv2d_pointwiser  binaryr  qlinear_pointwiser/  r  binary_tensorr  linear_dynamic_fp16linear_relu_dynamic_fp16r  r	  r  
max_pool2dr  int4mm_packed_weight_cpur'  r-  
avg_pool2dr[  r`  avg_pool2d_backwardri  
avg_pool3dr  avg_pool3d_backwardr  _adaptive_avg_pool2dr  _adaptive_avg_pool3dr  _adaptive_avg_pool2d_backwardr  _adaptive_avg_pool3d_backwardr  r  adaptive_max_pool2dr  r  r  adaptive_max_pool3dr  r  r  repeat_interleaver  rg   r  r  r  r   _unsafe_indexr  convolution_backwardr  addbmmr  _fused_adam__fused_adamw_r1  _fused_adamr9  _int_mmrD  _convert_weight_to_int4packrQ  #_convert_weight_to_int4pack_for_cpurW  _weight_int4pack_mmrb  _weight_int4pack_mm_for_cpuri  rk  r  _dyn_quant_pack_4bit_weightr  _dyn_quant_matmul_4bitr  _weight_int8pack_mmr  _cdist_forwardr  _cdist_backwardr  _embedding_bagr   _embedding_bag_forward_onlyr  r  nansumr  median	nanmedianr	  
dim_valuesrf  r   r  logical_not_r  repeatr  zero_r  mul_Scalardiv_logical_and_logical_or_logical_xor_r  add_sub_r'  rounddecimalsr-  r4  
__rshift__r8  
__lshift__r<  zeror>  r!  rA  fillrC  relu_rF  	_add_relurH  rrelu_with_noiserM  rrelu_with_noise_functionalrO  rrelu_with_noise_rQ  	index_put_unsafe_index_putrU  masked_fill_rY  _masked_scaler\  masked_scatter_ra  masked_scatterrc  masked_scatter_backwardre  
index_put_rg  aliasrj  rv  bmmrx  r|  r  rO  rP  r|  r  r  r   max_pool2d_with_indices_backwardr  max_pool2d_with_indicesr  fractional_max_pool2dr  max_pool3d_with_indicesr   max_pool3d_with_indices_backwardr  r  r  r  grid_sampler_2d_backwardr  r  r  r  r  r  r)  select_scatterr-  slice_scatterr1  r   r6  r?  gatherrF  rW  r_  ra  rl  ro  scatter_addrt  scatter_add_rw  rn  r(  rX  rf  value_reducer{  scatter_r}  #_scaled_dot_product_flash_attentionr  #_scaled_dot_product_cudnn_attentionr  0_scaled_dot_product_fused_attention_overrideabler  ,_scaled_dot_product_flash_attention_backwardr  +_scaled_dot_product_flash_attention_for_cpur  4_scaled_dot_product_flash_attention_for_cpu_backwardr  '_scaled_dot_product_efficient_attentionr  0_scaled_dot_product_efficient_attention_backwardr  ,_scaled_dot_product_cudnn_attention_backwardr  _flash_attention_forwardr  _flash_attention_backwardr  _efficient_attention_forwardr  _efficient_attention_backwardSymIntr  
_scaled_mmr  scatter_reducetwotwo_outr  scatter_reduce_r  multinomialr  r  r)  r2  _upsample_nearest_exact1dr<  _upsample_nearest_exact2drE  "_upsample_nearest_exact2d_backwardrJ  _upsample_nearest_exact3dr   rL  values_stablerO  rh  _thnn_fused_lstm_cellrn  r  r  r  r  r  argminr  r  topkr  _segment_reduce_backwardr  kthvaluer  r   r  r  r  r  pixel_shuffler  r  	bucketize
Tensor_outr  histcr  _upsample_bilinear2d_aa_upsample_bicubic2d_aar   _upsample_bilinear2d_aa_backwardr	  r	  r	  r  r	  searchsortedr#	  r)	  _embedding_bag_backwardr/	  _embedding_bag_dense_backwardr,	  *_embedding_bag_per_sample_weights_backwardr7	  isinr?	  	polygammarC	  _local_scalar_denserE	  rG	  rI	  _softmaxrM	  rQ	  _jagged_to_padded_dense_forwardrU	  r^	  rc	  special_airy_aispecial_bessel_y0special_bessel_y1special_modified_bessel_i0special_modified_bessel_i1special_modified_bessel_k0special_modified_bessel_k1!special_scaled_modified_bessel_k0!special_scaled_modified_bessel_k1special_chebyshev_polynomial_tspecial_chebyshev_polynomial_uspecial_chebyshev_polynomial_vspecial_chebyshev_polynomial_w&special_shifted_chebyshev_polynomial_t&special_shifted_chebyshev_polynomial_u&special_shifted_chebyshev_polynomial_v&special_shifted_chebyshev_polynomial_wspecial_hermite_polynomial_hspecial_hermite_polynomial_hespecial_laguerre_polynomial_lspecial_legendre_polynomial_prj	  ro	  rt	  rx	  lerp_addcmul_addcdiv_torch._refs.nn.functionaltorch._refs.specialr	  r=   r8   r5   <module>r
     stD    $   5 5 '  # + +  " U    < 7 ) T]t_yy~~*/--*?*?PV*W ' %a )X
8BF#3"4hr2v6F"FG 
3(* t}}-.
 

==5  /5p 		!!499==12'  3' !!))4+<+<+@+@AB%' %  C%$ t%%&I  'I 	[[$++//4;;+>+>P Xy! " !!))4+<+<+@+@ABI  CI3lV $s) 4  %%t}}'8'89:K  ;K $s)  %%t}}'8'89:=
  ;=
@ t}}**+"& 3 ,3 t}}$$% **
 &
 $$dll&6&678
 **  9&   $,,"6"678 **  9& 		!!499==12%)$tPT   3 %%t}}'8'89:$Dv $DDI $Dc $DC $D  ;$DN tzz!!" #0( t&&' ( t223
 "%)'+  6
	
 c] $ 4B t../
 (,	
  $	 00 t112 	
'+
  	 $ 3D t##$ ""'+"$-<,,-<\\-< 6
-< F	-<
 $-< -< -< -< -< %-<` t  (() 	I
	I		I 	I LL		I
 	I 	I 	I *	I t!!))* 	
			 	 LL		
 	 	 	 +	 t  (()' * ' t""**+
 !% $ $ W
 W W f	 W
 f W f W  W  W  W , WF   $(("4"456  7 txx||    $(("4"456  7 txx||  tzz!!"6 #6 tzz~~( (
 t!!))* + t!!%%& ' t{{""# $ t##++, ) -) t''//0, 1, t33;;< =
 t008896 :6& t<<DDE F
 t,,001 2
F C    F  #  N (,


 !%
$V S C 
 
F 
$ 
 
"  	  	C  !!))4+<+<+H+HIJ]N+ s T  , K" $$,,d.A.A.E.EFGB B6 B  HB  !]N+	6 	 , "	Q QF Q t**+) )F )4 )F )  ,) t""#J JF J4 JF J  $J t}})6 )$ )6 )  ) t$$%)6 )$ )6 )  &) t&&../&  T  0" 	$$,,d.M.M.Q.QR .f .6 .f . .d t!!))*&   + ))1143L3L3P3PQRT8V$ 	
  	
 666!" % S& %%--t/D/D/H/HIJ ''' '
 ' '  K'T &&(:(:;<S#s/3 f  fff>T8U   =4 ((00$2J2J2N2NOPT8V$ 	  	
 666!" % QD $$,,d.B.B.F.FGH 444 4
 4 4 4  I4n t~~S#s 	$$$ $ 	$
 666!"$  $P tTz!2 * &&(:(:;<S#f C ffn8M   =4 $$,,d.B.B.G.GHIV[$1'v '%(F"G ' 2 J'$ t''(   	""" " SM	" )"J.
.
. 49d3i .".
.
. 3-. 66>	.(f V   t$$%
 ##!777 	7
 7 V7 	7 V7 6
7 6666)*7 &7t ,,44d6R6R6V6VWX   	
   
&	  Y2 t$$%S#4( +(
+(+( +( 	+(
 +( 66>+( ) &+(^ t''(
 )
 tzz
 WW	W W 	W
 W W  Wt>#;L t$$%=  &= t%%&>  '>(< t--.\S  /S t../\T  0T2Ej t$$%=  &= t%%&>  '> &&..&&11''//''22	 \& &:<G~ t$$%=  &= t%%&>  '> &&..&&11''//''22	 \$( $(N t""#

f 

 

v 

  $

 t##$Pv PV P Pf PQW P  %P $$dll&6&678/0 '  9'2 &&(:(:;<&* I  =I
 t$$% & t~~ I !I
 $$dll&6&678"  9" t33;;< * =*. tww	  	B
* 7;h,,hLLh $s)S.!h 49c>"	h
 DIsN#h h h U49c>23hVQ t%%--."$,,"$LL"$ 5<<
 "$ 5<<(	"$
 %,,'"$ "$ !&"$ "$ /"$J t''(),,)LL) ,,) I	)
 #Y) 3i) ) I) ) ))X 	889>9N9N&&:6 599##::BBC D, 599##55==>S ?S
 xx:?--:O:O66;
7 
uyy}}00	1	 
2	
 :?9N9N&&:6 599##55==>  ? D 599##55<<= >6 599##55==>599##55<<= > ?, 599##55<<=599##55CCD E >: 599##77??@599##<<DDE	 F A	 =BMM<Q<QVV=9 599&&112 
 3
8 599&&??@@ A@( t&&' M (Mb(<X t''//0E 1EP t UJ   UJp t''(\K(  )K(\ t((001 2" t((001@ 2@ t1199:F ;F, t112\P  3P
	
6 	
S 	
 t''(UI+  )+\ t001\H  2H$ t''(UI'  )'T t001\(  2(
 t%%,,-* .* $$dll&6&678T  9T ##++T-@-@-D-DEF46 @c @  G@ 		&&..		0F0F0J0JKL  M" 

!!4#5#5#<#<=> ?D ))1123H 4H: ##T[[__56./q '  7'0 !!))4+=+=+E+EFG  !
 H
2   (()*  !! +!H ~B  B* 0012 3& 889: ; (()*@ +@ 0012< 3<"3 "3 "3 "jZ 0012D!)&!1D 3D0 ++,-; .;  (()*< +< t""**+ & , &F t##$G  %G* t""**+
 	
`5 ,`5F t//7785 95
 ##T[[__56=$ =  7= ##T^^%;%;<=) >) !!					 Xy! "	 t  (() * t{{""#' $' tzz!!" # 								!!  !!

 									**Z 

""DJJ$7$789 :
" &&(>(>?@ A &&(>(>?@ A tyy  !& "& 

!!4::#4#456 7 		  $))"2"234" 5" tzz!!" # t~~$$%F   & %%&'RV"  (" 0012RV; 3; &&'(KO ) &&(>(>(F(FGH" I" t  ''( )
 t!!))* + t##$	 %	 t""#6  $6 t++,! -! t&&' ( tzz!!"! #!B txx 5 !56;h #-YYY 	Y 		Y
 	Y 	Y 	Y 	Y 	Y 	Y 	Y Y Y Y Y  !Y" #Y$ %Y& 'Y( )Y* +Y, -Yx;4|383838 38 		38
 	38 	38 	38 	38 	38 	38 	38 	38 38 38 38  !38" #38$ %38& '38lI2X t44<<=( >(V t++334 # 5#L t))112Q 3Qh t++,UI d  -dN t445\b  6bJ%
V %
6 %
Pt  v 3 $ t,,445# 6#$ t##$8  %8" t,,-\;'! ( .!, 		!!"#. $. t&&' ) ()X t{{E  E@ t""**+. ,. t!!))*. +.
	 	C 	d 	/
  t{{""#' $'6
 
-
b4 t''(& )&
 t  ! "
 !!	 & & ""	 889:
 #!==	= = 	=
 = = E?= ;=@ 889: #!&&	& & 	&
 & & & & E?& ;&R EEFG
 #'#!%%	% % 	%
 % % % E?% H%P 99( """" 
" 	"
 
" " " " " " " " " " E?"
". 88 "&!	  	
   E?
: AA #'!'"'"'" 
'" 	'"
 
'" '" '" '" '" E?'"
'"T <<=> !))))	)) )) 	))
 )) )) E?)) ?))X ==" !4-4-4- 
4- 	4-
 4- 
4- 4- 4- 4- 4- $Z4- 4- E?4-
4-n 99* "!""" 
" 	"
 
" " " " " " " " " " "  E?!"
"0 %%  "&*'+"&%)HH	H H 	H
 H H H H H H E?H smH  }H H 6"H
HV &&( "&*'+#,,, 
, 	,
 
, , , , , , , , , , E?,  sm!,"  }#,
,4 ))   %!(,!%!%,S,S	,S ,S 6
	,S
 6",S 6",S 3-,S 3-,S ,S ,S ,S E?,S f%,S v,S #,S
,S^ *** "$("'%474747 
47 	47
 6
47 6"47 6"47 ,,47 ,,47 47 47 47 47 47 47  E?!47" SM#47$  %47
47n ''() $(+/'+ aY
,,aY
,,aY \\aY \\	aY
 5<<
 aY 5<<(aY $aY aY *aYH ##'')<)<)D)DEF&  G&
 t##''( )
   (($*:*:*>*>?@	 	  A	,* 	$$d&D&D&L&LM

 	$$d&D&D&L&LM. ((00//77 !% $%U\\ 123 sELL012 uo	
 uo: 	$$d&D&D&L&LM

 									&$N t))112
  3 t&&'4/ (4/n t$$,,-$% .$%N


 ##T[[%8%89:4 ;4 t!!))* + tyy  !
Q "
Q t,,-LP  .  %%t}}';';<=Xy!K " >K #("9"9 Q t77??@	* A	* t##++,	0 -	0 t!!))* +> t--556F 7FD %%t~~'@'@AB27u   C 

|E  E0 	!!))4+F+F+N+NO & 55==>?  @8 t>>FFG H$ '')<)<=>'  ?'
 uyy~~(() *8 uyy~~  !6 "6$ t  !
 
	-K  "-K` t++, '
 -'
T t112  3, t>>?  @. tyy8=e 8  8  t~~6c 6 6F 6  6 t''(I& I )I tyy"v "& "  " t||6& 6V 6  6 t}}	v 	C 	 	 	  	 t~~ $888 8 	8
 8 8  8, t33;;<
 	**&\* c* 	* =*  d22 3 d44 5 d44 5 d== > d== > d== > d== > dDD E dDD E tBB C tBB C tBB C tBB C tJJ K tJJ K tJJ K tJJ K t@@ A tAA B tAA B tAA B tyy  $ t||./    t||./   , 	tyy)!$,,/!$,,/
    BNJ r8   