o
    wZh                    @   sD  U d dl mZ d dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlZd dlZd dlmZ d dlm  mZ d dlmZ d dlmZmZmZmZmZmZmZmZ d d	lm Z m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)Zd dl*m+Z+m,Z,m-Z- d dl.m/Z/ ej0Z0dZ1dd Z2dd Z3e4dddddd ddej5de4dddddd d ddej5d!	e4d"d#d$ej5d%e4d"d&d'd(d)d$ej5d*e4d+d,d-d ej5d.e4d/d0d1d2d$d$ej5d3gZ6d4d5 Z7d6d7 Z8d8d9 Z9d:d; Z:d<d= Z;d>d? Z<d@dA Z=dBdC Z>dDdE Z?dFdG Z@dHdI ZAdJdK ZBdLdM ZCdNdO ZDdPdQ ZEdRdS ZFdTdU ZGdVdW ZHdXdY ZIdZd[ ZJd\d] ZKd^d_ ZLd`da ZMdbdc ZNddde ZOdfdg ZPdhdi ZQdjdk ZRdldm ZSdndo ZTdpdq ZUdrds ZVdtdu ZWdvdw ZXdxdy ZYdzd{ ZZd|d} Z[d~d Z\dd Z]dd Z^dd Z_dd Z`dd Zadd Zbdd Zcdd Zddd Zedd Zfdd Zgdd Zhdd Zidd Zjdd Zkdd ZldddZm		dddZn		dddZo		dddZp		dddZq		dddZrdddZsdddZtdd ZudddZvdddZwdddZxdd ZydddZzdddZ{	$	dddƄZ|dddȄZ}dddʄZ~emeemddˍerenesetevewexeze{e|e}e~eqd̜Ze4de+f ed< g ZddЄ Zg dѢZg dҢZe
eeD ] \ZZe4e de edԜdddded$ej5d׍Zee q[eD ]Ze4de dd dd dd ed$ej5d܍Zee q~ddd dd fddd dd fddd dd fddd dd fddd dd fddd dd fddd dd fddd dd fddd dd fddd dd fg
Zddd$iiZe4ee4f ed< d$d$d$d$d$dZg dҢZe
eeD ]:\\ZZZZe4e de edԜddefddefddedeedd Zeei Zee ee qG dd deZG dd dZG dd deZG dd dZG d	d
 d
eeZG dd deeZdddZd ddZdd ZdS (!      )abstractmethodN)deepcopy)reducepartial)product)mul)
_reduction)TestCaseto_gpufreeze_rng_stateis_iterable	gradcheckgradgradcheckset_default_dtypeskipIfTorchDynamo)	TEST_CUDASM90OrLater)_get_numerical_jacobian_iter_tensors)Variable)_TensorOrTensors)CallableUnionAny)Sequenceh㈵>c                 C   s<   t | dd }|d u rtjt | dd ddd}|d usJ |S )N	reductionZsizeAverageTF)Zemit_warning)getattr
_ReductionZlegacy_get_stringmresult r"   P/var/www/auris/lib/python3.10/site-packages/torch/testing/_internal/common_nn.pyget_reduction!   s
   r$   c                 C   s$   t | dd }|d ur|S t | dd S )Nweightweights)r   r   r"   r"   r#   
get_weight)   s   r'   Linear)
      ztorch::nn::LinearOptions(10, 8))   r)   c                 C   s,   t | |d  |d dddd S )Nr      r+   r*   )torchmmtviewexpandip_r"   r"   r#   <lambda>l   s   , r7   T{Gzt?)module_nameconstructor_argscpp_constructor_args
input_sizereference_fn	with_tf32tf32_precisiondefault_dtype)r)   r*   Fz+torch::nn::LinearOptions(10, 8).bias(false)no_biasc                 C   s   t | |d  S )Nr   )r.   r/   r0   r3   r"   r"   r#   r7   w       )	r9   r:   r;   r<   descr=   r>   r?   r@   RReLU)r,      rE   F)r9   r<   	test_cudar@   皙?g?/torch::nn::RReLUOptions().lower(0.1).upper(0.9))r+   r+      Zwith_up_downr9   r:   r;   r<   rC   rF   r@   FlattenrE      r+   rJ   c                 G   s   t | dS Nr,   )r.   flattenr4   r6   r"   r"   r#   r7          )r9   r<   r=   r@   ZCrossMapLRN2d)rJ   r8   MbP?rE   z>torch::nn::CrossMapLRN2dOptions(5).alpha(5e-3).beta(1e-3).k(2)rE   rN      rU   )r9   r:   r;   r<   check_gradgradcheck_batched_gradr@   c                  G   s    t t| d}t|j|   S rO   )r   r   r.   randpermr1   double)sizetotalr"   r"   r#   _rand_tensor_non_equal   s   r\   c                    s   G  fdddt j}|S )Nc                       s   e Zd Z fddZdS )z)wrap_functional.<locals>.FunctionalModulec                    s    |i S Nr"   )selfargsfnkwargsr"   r#   forward   s   z1wrap_functional.<locals>.FunctionalModule.forwardN)__name__
__module____qualname__rc   r"   r`   r"   r#   FunctionalModule   s    rg   )nnModule)ra   rb   rg   r"   r`   r#   wrap_functional   s   rj   c                
      sD   t dd tdt fddddd d d fd	dd
t jdS )Nr)   ZPoissonNLLLoss_no_reducec                       t j|  | ddS Nnoner   )FZpoisson_nll_losstype_asr4   r0   r"   r#   r7          z/poissonnllloss_no_reduce_test.<locals>.<lambda>zaF::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(torch::kNone))c                   S      t ddS Nr)   r.   randr"   r"   r"   r#   r7      rR   _get_input()r4   r0   c                    s   |    |  S r]   )expr   rQ   rr   r"   r#   r7          Ffullnameconstructorcpp_function_callinput_fncpp_var_mapr=   pickler@   r.   randndictrj   rY   r"   r"   rr   r#   poissonnllloss_no_reduce_test   s   

r   c                      sX   t tdddtj tdt fddddd d	 d
 fddddtjd	S )N   r)   r   ZBCELoss_no_reducec                    rk   rl   ro   Zbinary_cross_entropyrp   rq   rr   r"   r#   r7      rs   z(bceloss_no_reduce_test.<locals>.<lambda>iF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone))c                   S      t ddddS Nr   r)   y&1?v?r.   rw   clamp_r"   r"   r"   r#   r7      rB   rx   ry   c                    "    |    d  d|       S rO   logrQ   rr   r"   r#   r7         " FgǺF?	r}   r~   r   r   r   r=   r   	precisionr@   )r   r.   r   gttorY   r   rj   r"   r"   rr   r#   bceloss_no_reduce_test   s   

r   c                
      sP   t ddt j tdt fddddd d d	 fd
ddt jdS )Nr"   r   ZBCELoss_no_reduce_scalarc                    rk   rl   r   rq   rr   r"   r#   r7      rs   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>r   c                   S      t dddS Nr"   r   r   r   r"   r"   r"   r#   r7      r{   rx   ry   c                    r   rO   r   rQ   rr   r"   r#   r7      r   Fr|   )r.   r   r   r   rY   r   rj   r"   r"   rr   r#   bceloss_no_reduce_scalar_test   s   

r   c                      st   t tjddtjddtj tjdtjdtdt fdddd	d d
 d fddddtjd	S )Nr   r)   dtyper   ZBCELoss_weights_no_reducec                       t j|  | | ddS Nrm   r%   r   r   rq   r0   r&   r"   r#   r7          
z0bceloss_weights_no_reduce_test.<locals>.<lambda>zF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                   S   r   r   r   r"   r"   r"   r#   r7      rB   rx   r4   r0   r&   c                    &    |    d  d|        S rO   r   )r4   r5   r    r   r"   r#   r7         & Fa2U0*3?r   )	r   r.   r   rY   r   r   rw   r   rj   r"   r"   r   r#   bceloss_weights_no_reduce_test   s   $
r   c                
      sf   t ddt j t jdt jdtdt fdddd d	d
d  fdddt jdS )Nr"   r   r   Z BCELoss_weights_no_reduce_scalarc                    r   r   r   rq   r   r"   r#   r7      r   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>zF::binary_cross_entropy(
            i, t.to(i.options()),
            F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))rx   r   c                   S   r   r   r   r"   r"   r"   r#   r7      r{   c                    r   rO   r   rQ   r   r"   r#   r7      r   F)r}   r~   r   r   r   r=   r   r@   )r.   r   r   r   rY   rw   r   rj   r"   r"   r   r#   %bceloss_weights_no_reduce_scalar_test   s   
r   c                      b   t tdddtjt  tdt	fddddd d	d
 fddddtjd	S )Nr   r)   r   ZBCEWithLogitsLoss_legacy_enumc                    rk   )NF)r   ro   Z binary_cross_entropy_with_logitsrp   rq   rr   r"   r#   r7   	  rs   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>F::binary_cross_entropy_with_logits(
            i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(torch::kNone))c                   S   r   r   r   r"   r"   r"   r#   r7     rB   rx   ry   c                    *    |    d d |       S rO   r   rQ   sigmoidr0   r"   r#   r7        * F	r}   r~   r   r   r   r=   rV   r   r@   
r   r.   r   r   r   rY   rh   Sigmoidr   rj   r"   r"   r   r#   "bce_with_logistic_legacy_enum_test     
r   c                      r   )Nr   r)   r   ZBCEWithLogitsLoss_no_reducec                    rk   rl   r   rq   rr   r"   r#   r7     rs   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>r   c                   S   r   r   r   r"   r"   r"   r#   r7     rB   rx   ry   c                    r   rO   r   rQ   r   r"   r#   r7      r   Fr   r   r"   r"   r   r#    bce_with_logistic_no_reduce_test  r   r   c                      s\   t ddt jt  tdtfddddd dd	 fd
dddt jd	S )Nr"   r   Z"BCEWithLogitsLoss_no_reduce_scalarc                    rk   rl   r   rq   rr   r"   r#   r7   -  rs   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>r   c                   S   r   r   r   r"   r"   r"   r#   r7   0  r{   rx   ry   c                    r   rO   r   rQ   r   r"   r#   r7   2  r   Fr   )	r.   r   r   r   rY   rh   r   r   rj   r"   r"   r   r#   'bce_with_logistic_no_reduce_scalar_test'  s   
r   c                      L   t jddt jd tdt fddddd d d	 fd
dddt jd	S )Nr)   r   ZKLDivLoss_with_target_no_reducec                    rk   rl   ro   Zkl_divrp   rq   rr   r"   r#   r7   >  rs   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>NF::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone))c                   S      t dd S ru   r.   rw   r   r"   r"   r"   r#   r7   @      rx   ry   c                       t d |  | ddS N	KLDivLossrm   rn   loss_reference_fnsrp   rQ   rr   r"   r#   r7   B     TF	r}   r~   r   r   r   r=   supports_forward_adr   r@   r.   rw   rY   r   rj   r"   r"   rr   r#   $kldivloss_with_target_no_reduce_test9     

r   c                      r   )Nr)   r   ZKLDivLoss_no_reducec                    rk   rl   r   rq   rr   r"   r#   r7   N  rs   z*kldivloss_no_reduce_test.<locals>.<lambda>r   c                   S   r   ru   r   r"   r"   r"   r#   r7   P  r   rx   ry   c                    r   r   r   rQ   rr   r"   r#   r7   R  r   TFr   r   r"   r"   rr   r#   kldivloss_no_reduce_testI  r   r   c                      J   t jdt jd tdt fddddd d d	 fd
dddt jd	S )Nr"   r   ZKLDivLoss_no_reduce_scalarc                    rk   rl   r   rq   rr   r"   r#   r7   _  rs   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>r   c                   S      t d S Nr"   r   r"   r"   r"   r#   r7   a      rx   ry   c                    r   r   r   rQ   rr   r"   r#   r7   c  r   TFr   r   r"   r"   rr   r#   kldivloss_no_reduce_scalar_testZ  s   

r   c                      P   t jddt jd  tdt fddddd d d	 fd
dddt jd	S )Nr)   r   Z#KLDivLoss_with_log_target_no_reducec                       t j|  | dddS Nrm   T)r   
log_targetr   rq   rr   r"   r#   r7   o      z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>_F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone).log_target(true))c                   S   r   ru   r   r"   r"   r"   r#   r7   q  r   rx   ry   c                    r   NKLDivLoss_log_targetrm   rn   r   rQ   rr   r"   r#   r7   s  r   TFr   r.   rw   rY   r   r   rj   r"   r"   rr   r#   (kldivloss_with_log_target_no_reduce_testj     

r   c                      r   )Nr)   r   ZKLDivLoss_no_reduce_log_targetc                    r   r   r   rq   rr   r"   r#   r7     r   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>r   c                   S   r   ru   r   r"   r"   r"   r#   r7     r   rx   ry   c                    r   r   r   rQ   rr   r"   r#   r7     r   TFr   r   r"   r"   rr   r#   #kldivloss_no_reduce_log_target_testz  r   r   c                      sN   t jdt jd  tdt fddddd d d	 fd
dddt jd	S )Nr"   r   Z%KLDivLoss_no_reduce_scalar_log_targetc                    r   r   r   rq   rr   r"   r#   r7     r   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>r   c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                    r   r   r   rQ   rr   r"   r#   r7     r   TFr   r   r"   r"   rr   r#   *kldivloss_no_reduce_scalar_log_target_test  s   

r   c                      N   t jdddt jd tdt fdddd	d d
 d fddddt jd	S )NrE   rN   r+   r   ZL1Loss_no_reducec                    rk   rl   ro   Zl1_lossrp   rq   rr   r"   r#   r7     rs   z'l1loss_no_reduce_test.<locals>.<lambda>PF::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(torch::kNone))c                   S      t dddS NrE   rN   r+   r.   r   r"   r"   r"   r#   r7     r   rx   ry   c                       |   |   S r]   rp   absrQ   rr   r"   r#   r7     r{   TFr   r.   r   rY   r   rj   r"   r"   rr   r#   l1loss_no_reduce_test  s   

r   c                
      sJ   t jdddt jd tdt fdddd	d d
 d fdddddS )NrE   rN   r+   r   ZL1Loss_no_reduce_complexc                    rk   rl   r   rq   rr   r"   r#   r7     rs   z/l1loss_no_reduce_complex_test.<locals>.<lambda>r   c                   S   s   t jdddt jdS )NrE   rN   r+   r   )r.   r   cdoubler"   r"   r"   r#   r7     rB   rx   ry   c                    r   r]   r   rQ   rr   r"   r#   r7     r{   TF)r}   r~   r   r   r   r=   r   r   )r.   r   r   r   rj   r"   r"   rr   r#   l1loss_no_reduce_complex_test  s   

r   c                      r   )Nr"   r   ZL1Loss_no_reduce_scalarc                    rk   rl   r   rq   rr   r"   r#   r7     rs   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>r   c                   S   
   t dS r   r   r"   r"   r"   r#   r7        
 rx   ry   c                    r   r]   r   rQ   rr   r"   r#   r7     r{   TFr   r   r"   r"   rr   r#   l1loss_no_reduce_scalar_test  s   

r   c                     sL   d} t j| dt ji tdt fddd| d d fd	dd
dt jd	S )NrM   r   ZMSELoss_no_reducec                    rk   rl   ro   Zmse_lossrp   rq   targetr"   r#   r7     rs   z(mseloss_no_reduce_test.<locals>.<lambda>WF::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(torch::kNone))rx   r4   r   c                       |    dS NrE   powrQ   r   r"   r#   r7     r   TF	r}   r~   r   r<   r   r=   r   r   r@   r   r<   r"   r   r#   mseloss_no_reduce_test  s   

r   c                     sJ   d} t j| t jd tdt fddd| d d fd	dd
dt jd	S )Nr"   r   ZMSELoss_no_reduce_scalarc                    rk   rl   r   rq   r   r"   r#   r7     rs   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>r   rx   r   c                    r   r   r   rQ   r   r"   r#   r7     r   TFr   r   r   r"   r   r#   mseloss_no_reduce_scalar_test  s   

r   c                
      sd   t td d  ddi tdt fdddd	d d
d fdddtj	dS )Nr   r)   r   rm   ZNLLLoss_no_reducec                       t j| |   d dS Nr   rn   ro   nll_lossrp   longrq   rb   r0   r"   r#   r7         z(nllloss_no_reduce_test.<locals>.<lambda>pF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().reduction(torch::kNone))c                   S      t dd S Nr   r)   r   r"   r"   r"   r#   r7     r   rx   ry   c                        t d | |  fi  S NNLLLossr   rp   r   rQ   r   r"   r#   r7         Fr|   
r   r.   emptyuniform_r   floorr   r   rj   rY   r"   r"   r   r#   nllloss_no_reduce_test      r  c                
      sf   t td d  ddd tdt fddd	d
d dd fdddtj	dS )Nr   r)   rE   rm   ignore_indexr   ZNLLLoss_no_reduce_ignore_indexc                    ,   t j| |  t d t d dS Nr  r   r  ro   r   rp   r   intstrrq   r   r"   r#   r7         
z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(2).reduction(torch::kNone))c                   S   r   r  r   r"   r"   r"   r#   r7     r   rx   ry   c                    r  r  r  rQ   r   r"   r#   r7     r  Fr|   r  r"   r"   r   r#   #nllloss_no_reduce_ignore_index_test      
r  c                
      t   t td d  tdfdd tdt	 fdddd	d d
d fdddtj
dS )Nr   r)   c                         | ddS r   rp   rq   r%   r"   r#   rb        z.nllloss_no_reduce_weights_test.<locals>.kwargsZNLLLoss_no_reduce_weightsc                    "   t j| |  fi  | S r]   r   rq   r   r"   r#   r7     r   z0nllloss_no_reduce_weights_test.<locals>.<lambda>F::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone))c                   S      t ddd S Nr   r)   {Gz?r.   rw   addr   r"   r"   r"   r#   r7     rs   rx   r4   r0   r%   c                    $   t d | |  fi  | S r  r  rQ   r   r"   r#   r7        $Fr|   r   r.   r  r	  r   r
  r   rw   r   rj   rY   r"   r"   rb   r0   r%   r#   nllloss_no_reduce_weights_test
      

r(  c                
      r  )Nr   r)   c                         | dddS )Nrm   rE   r%   r   r  r  rq   r  r"   r#   rb   $     
z;nllloss_no_reduce_weights_ignore_index_test.<locals>.kwargsZ&NLLLoss_no_reduce_weights_ignore_indexc                    s$   t j| |  fi  | jS r]   )ro   r   rp   r   datarq   r   r"   r#   r7   +  s   $ z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(2))c                   S   r  r  r!  r"   r"   r"   r#   r7   /  rs   rx   r#  c                    r$  r  r  rQ   r   r"   r#   r7   1  r%  Fr|   r&  r"   r"   r'  r#   +nllloss_no_reduce_weights_ignore_index_test   s    

r.  c                
      s   t td d  tdfdd tdt	 fdddtjddtj
d	d
 dd fdddtj
dS )Nr   r)   c                    r*  )Nrm   r-   r+  r  rq   r  r"   r#   rb   ;  r,  z?nllloss_no_reduce_weights_ignore_index_neg_test.<locals>.kwargsZ*NLLLoss_no_reduce_weights_ignore_index_negc                    r  r]   r   rq   r   r"   r#   r7   B  r   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(-1))r   r   rx   r#  c                    r$  r  r  rQ   r   r"   r#   r7   H  r%  F)r}   r~   r   inputr   r=   r   r@   )r   r.   r  r	  r   r
  r   rw   r   rj   rY   r"  r   r"   r"   r'  r#   /nllloss_no_reduce_weights_ignore_index_neg_test7  s    

r0  c                
      sd   t tdddd  ddi tdt fddd	d
d dd fdddtjdS )NrE   rJ   rN   r   rm   ZNLLLoss2d_no_reducec                    r   r   r   rq   r   r"   r#   r7   T  r   z*nllloss2d_no_reduce_test.<locals>.<lambda>r   c                   S      t dddd S NrE   rN   rJ   r   r"   r"   r"   r#   r7   W  rB   rx   ry   c                    r  N	NLLLossNdr  rQ   r   r"   r#   r7   Y  r  Fr|   	r   r.   rw   r   r
  r   r   rj   rY   r"   r"   r   r#   nllloss2d_no_reduce_testN  r  r6  c                
      sf   t tdddd  ddd tdt fdd	d
dd	 dd fdd	dtjdS )NrE   rJ   rN   r,   rm   r  Z NLLLoss2d_no_reduce_ignore_indexc                    r  r  r  rq   r   r"   r#   r7   e  r  z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>F::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(torch::kNone))c                   S   r1  r2  r   r"   r"   r"   r#   r7   i  rB   rx   ry   c                    r  r3  r  rQ   r   r"   r#   r7   k  r  Fr|   r5  r"   r"   r   r#   %nllloss2d_no_reduce_ignore_index_test_  r  r8  c                
      st   t tdddd  tdfdd tdt fddd	d
d dd fdddtjdS )NrE   rJ   rN   c                    r  r   r  rq   r  r"   r#   rb   u  r  z0nllloss2d_no_reduce_weights_test.<locals>.kwargsZNLLLoss2d_no_reduce_weightsc                    r  r]   r   rq   r   r"   r#   r7   {  r   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>r  c                   S   r1  r2  r   r"   r"   r"   r#   r7     rB   rx   r#  c                    r$  r3  r  rQ   r   r"   r#   r7     r%  Fr|   r5  r"   r"   r'  r#    nllloss2d_no_reduce_weights_testq  r)  r9  c                
      sh   t tdddddd  ddi tdt fddd	d
d dd fdddtjdS )NrE   rJ   rN   r   rm   ZNLLLossNd_no_reducec                    r   r   r   rq   r   r"   r#   r7     r   z*nlllossNd_no_reduce_test.<locals>.<lambda>r   c                   S      t dddddd S r2  r   r"   r"   r"   r#   r7     r   rx   ry   c                    r  r3  r  rQ   r   r"   r#   r7     r  Fr|   r5  r"   r"   r   r#   nlllossNd_no_reduce_test  s   $r;  c                
      sj   t tdddddd  ddd tdt fdd	d
dd	 dd fdd	dtjdS )NrE   rJ   rN   r,   rm   r  Z NLLLossNd_no_reduce_ignore_indexc                    r  r  r  rq   r   r"   r#   r7     r  z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>r7  c                   S   r:  r2  r   r"   r"   r"   r#   r7     r   rx   ry   c                    r  r3  r  rQ   r   r"   r#   r7     r  Fr|   r5  r"   r"   r   r#   %nlllossNd_no_reduce_ignore_index_test  s   $
r<  c                
      sx   t tdddddd  tdfdd tdt fddd	d
d dd fdddtjdS )NrE   rJ   rN   c                    r  r   r  rq   r  r"   r#   rb     r  z0nlllossNd_no_reduce_weights_test.<locals>.kwargsZNLLLossNd_no_reduce_weightsc                    r  r]   r   rq   r   r"   r#   r7     r   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>r  c                   S   r:  r2  r   r"   r"   r"   r#   r7     r   rx   r#  c                    r$  r3  r  rQ   r   r"   r#   r7     r%  Fr|   r5  r"   r"   r'  r#    nlllossNd_no_reduce_weights_test  s   $

r=  c                      r   )NrE   rN   r+   r   ZSmoothL1Loss_no_reducec                    rk   rl   ro   Zsmooth_l1_lossrp   rq   rr   r"   r#   r7     rs   z-smoothl1loss_no_reduce_test.<locals>.<lambda>jF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone))c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                    r   NSmoothL1Lossrm   rn   r   rQ   rr   r"   r#   r7     r   TFr   r   r"   r"   rr   r#   smoothl1loss_no_reduce_test     

rB  c                      r   )Nr"   r   ZSmoothL1Loss_no_reduce_scalarc                    rk   rl   r>  rq   rr   r"   r#   r7     rs   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>r?  c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                    r   r@  r   rQ   rr   r"   r#   r7     r   TFr   r   r"   r"   rr   r#   "smoothl1loss_no_reduce_scalar_test  s   

rD  c                      r   )NrE   rN   r+   r   ZSmoothL1Loss_betac                    r   )Nrm         ?r   betar>  rq   rr   r"   r#   r7     r   z(smoothl1loss_beta_test.<locals>.<lambda>zoF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0.5)c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                       t d |  | dddS )NrA  rm   rE  rF  r   rQ   rr   r"   r#   r7        TFr   r   r"   r"   rr   r#   smoothl1loss_beta_test  rC  rJ  c                      r   )NrE   rN   r+   r   ZSmoothL1Loss_zero_betac                    r   )Nrm   r   rF  r>  rq   rr   r"   r#   r7     r   z-smoothl1loss_zero_beta_test.<locals>.<lambda>zmF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0)c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                    rH  )NrA  rm   r   rF  r   rQ   rr   r"   r#   r7     rI  TFr   r   r"   r"   rr   r#   smoothl1loss_zero_beta_test  rC  rK  c                      sH   t ddd tdt fddddd d	 d
 fddddt jd	S )NrE   rN   r+   ZHuberLoss_deltac                    r   )Nrm   rE  r   delta)ro   Z
huber_lossrp   rq   rr   r"   r#   r7   	  r   z&huberloss_delta_test.<locals>.<lambda>znF::huber_loss(
            i, t.to(i.options()), F::HuberLossFuncOptions().reduction(torch::kNone).delta(0.5))c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                    rH  )N	HuberLossrm   rE  rL  r   rQ   rr   r"   r#   r7     rI  TFr   r   r"   r"   rr   r#   huberloss_delta_test  s   

rO  c                      sF   t d  tdt fddddd d d fd	dd
ddd	S )Nr"   Z!MultiLabelMarginLoss_0d_no_reducec                       t j|  |  ddS rl   ro   Zmultilabel_margin_lossrp   r   rq   rr   r"   r#   r7         z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>F::multilabel_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultilabelMarginLossFuncOptions().reduction(torch::kNone))c                   S   r   r   r   r"   r"   r"   r#   r7     r   rx   ry   c                       t d |  j|  ddS NMultiLabelMarginLossrm   rn   r   r-  rp   r   rQ   rr   r"   r#   r7        TF)	r}   r~   r   r   r   r=   check_sum_reductionrV   r   )r.   zerosr   r   rj   r"   r"   rr   r#   &multilabelmarginloss_0d_no_reduce_test  s   

r[  c                      sX   t tdd   tdt fddddd d d fd	dd
ddtjd
S )Nr)   Z!MultiLabelMarginLoss_1d_no_reducec                    rP  rl   rQ  rq   rr   r"   r#   r7   +  rR  z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>rS  c                   S   r   ru   r   r"   r"   r"   r#   r7   .  r   rx   ry   c                    rT  rU  rW  rQ   rr   r"   r#   r7   0  rX  TF
r}   r~   r   r   r   r=   rY  rV   r   r@   r5  r"   r"   rr   r#   &multilabelmarginloss_1d_no_reduce_test&  s   

r]  c                      sj   t tjtdddd  dd tdt	 fdd	d
dd	 d d fdd	dddtj
d
S )NrJ   r)   g         r-   minZMultiLabelMarginLoss_index_negc                    rP  rl   rQ  rq   rr   r"   r#   r7   =  rR  z5multilabelmarginloss_index_neg_test.<locals>.<lambda>rS  c                   S      t ddS NrJ   r)   r   r"   r"   r"   r#   r7   @  rR   rx   ry   c                    rT  rU  rW  rQ   rr   r"   r#   r7   B  rX  TFr\  )r   r.   clamprw   r"  r   r
  r   r   rj   rY   r"   r"   rr   r#   #multilabelmarginloss_index_neg_test8  s   .

rd  c                      sZ   t tddd   tdt fddddd d d	 fd
ddddtjd
S )NrJ   r)   ZMultiLabelMarginLoss_no_reducec                    rP  rl   rQ  rq   rr   r"   r#   r7   O  rR  z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>rS  c                   S   ra  rb  r   r"   r"   r"   r#   r7   R  rR   rx   ry   c                    rT  rU  rW  rQ   rr   r"   r#   r7   T  rX  TFr\  r5  r"   r"   rr   r#   #multilabelmarginloss_no_reduce_testJ  s   

re  c                      b   t tddtjdd tdt	 fdddd	d d
 d fddddtjd	S )Nr)   r   rE   r,   ZHingeEmbeddingLoss_no_reducec                    rk   rl   ro   Zhinge_embedding_lossrp   rq   rr   r"   r#   r7   a  rs   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>zvF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().reduction(torch::kNone))c                   S   r   ru   r   r"   r"   r"   r#   r7   d  r   rx   ry   c                    r   )NHingeEmbeddingLossrm   rn   r   rQ   rr   r"   r#   r7   f  r   TF	r}   r~   r   r   r   r=   rY  r   r@   
r   r.   r   r   r   rY   Zmul_subr   rj   r"   r"   rr   r#   !hingeembeddingloss_no_reduce_test\     (

rl  c                      rf  )Nr)   r   rE   r,   Z#HingeEmbeddingLoss_margin_no_reducec                    r   NrE  rm   marginr   rg  rq   rr   r"   r#   r7   r  r   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>zF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().margin(0.5).reduction(torch::kNone))c                   S   r   ru   r   r"   r"   r"   r#   r7   u  r   rx   ry   c                    rH  )Nrh  rE  rm   ro  r   rQ   rr   r"   r#   r7   w  rI  TFri  rj  r"   r"   rr   r#   (hingeembeddingloss_margin_no_reduce_testm  rm  rq  c                      r   )NrJ   r   ZSoftMarginLoss_no_reducec                    rk   rl   )ro   Zsoft_margin_lossrp   rq   rr   r"   r#   r7     rs   z/softmarginloss_no_reduce_test.<locals>.<lambda>znF::soft_margin_loss(
            i, t.to(i.options()), F::SoftMarginLossFuncOptions().reduction(torch::kNone))c                   S   rt   )NrJ   r   r"   r"   r"   r#   r7     rR   rx   ry   c                    r   )NSoftMarginLossrm   rn   r   rQ   rr   r"   r#   r7     r   TFr   r   r"   r"   rr   r#   softmarginloss_no_reduce_test~  s   

rs  c                      sP   t ddd  tdt fddddd d	 d
 fddddt jd	S )NrJ   r)   rE   Z"MultiLabelSoftMarginLoss_no_reducec                    rk   rl   ro   Zmultilabel_soft_margin_lossrp   rq   rr   r"   r#   r7     rs   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>zF::multilabel_soft_margin_loss(
            i, t.to(i.options()), F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone))c                   S   ra  rb  r   r"   r"   r"   r#   r7     rR   rx   ry   c                    s:    |     d  |        jdd| d S Nr,   dimr   r   sumrZ   rQ   rr   r"   r#   r7     s   :Fr   r.   rw   r   r
  r   rj   rY   r"   r"   rr   r#   'multilabelsoftmarginloss_no_reduce_test  s   

r{  c                      sb   t ddd  t dtdt fddddd d	 d
 fdddddt jd
S )NrJ   r)   rE   Z*MultiLabelSoftMarginLoss_weights_no_reducec                    r   r   rt  rq   r   r"   r#   r7     r   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>zF::multilabel_soft_margin_loss(
            i, t.to(i.options()),
            F::MultilabelSoftMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                   S   ra  rb  r   r"   r"   r"   r#   r7     rR   rx   r   c                    s>    |     d  |         jdd| d S ru  rx  rQ   r   r"   r#   r7     s   >TFr\  rz  r"   r"   r   r#   /multilabelsoftmarginloss_weights_no_reduce_test  s    

r|  c                      T   t dd   tdt fddddd d d	 fd
ddddt jd
S )NrJ   r*   ZMultiMarginLoss_no_reducec                    rP  rl   ro   Zmulti_margin_lossrp   r   rq   rr   r"   r#   r7     rR  z0multimarginloss_no_reduce_test.<locals>.<lambda>F::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().reduction(torch::kNone))c                   S   ra  rb  r   r"   r"   r"   r#   r7     rR   rx   ry   c                    rT  NMultiMarginLossrm   rn   rW  rQ   rr   r"   r#   r7     rX  TFr\  r.   rw   r   r
  r   r   rj   rY   r"   r"   rr   r#   multimarginloss_no_reduce_test     

r  c                      r}  )Nr,   r*   ZMultiMarginLoss_1d_no_reducec                    rP  rl   r~  rq   rr   r"   r#   r7     rR  z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>r  c                   S   r   ru   r   r"   r"   r"   r#   r7     r   rx   ry   c                    rT  r  rW  rQ   rr   r"   r#   r7     rX  TFr\  r  r"   r"   rr   r#   !multimarginloss_1d_no_reduce_test  r  r  c                      r}  )Nr"   r*   Z,multimarginloss_1d_input_0d_target_no_reducec                    rP  rl   r~  rq   rr   r"   r#   r7     rR  zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>r  c                   S   r   ru   r   r"   r"   r"   r#   r7     r   rx   ry   c                    rT  r  rW  rQ   rr   r"   r#   r7     rX  TFr\  r  r"   r"   rr   r#   1multimarginloss_1d_input_0d_target_no_reduce_test  r  r  c                      r}  )NrJ   r*   ZMultiMarginLoss_p_no_reducec                       t j|  |  dddS )NrE   rm   r5   r   r~  rq   rr   r"   r#   r7         z2multimarginloss_p_no_reduce_test.<locals>.<lambda>zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().p(2).reduction(torch::kNone))c                   S   r   )NrJ   r)   r   gGz?)r.   r   r   r"   r"   r"   r#   r7     rB   rx   ry   c                        t d |  j|  dddS )Nr  rE   rm   r  rW  rQ   rr   r"   r#   r7     r  TFr\  r  r"   r"   rr   r#    multimarginloss_p_no_reduce_test  r  r  c                      r}  )NrJ   r*   Z MultiMarginLoss_margin_no_reducec                    r  rn  r~  rq   rr   r"   r#   r7     r  z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().margin(0.5).reduction(torch::kNone))c                   S   ra  rb  r   r"   r"   r"   r#   r7     rR   rx   ry   c                    r  )Nr  rE  rm   ro  rW  rQ   rr   r"   r#   r7        TFr\  r  r"   r"   rr   r#   %multimarginloss_margin_no_reduce_test  s   

r  c                      sj   t dd   t jdt jdtdt fdddd	d d
 d fdddddt jd
S )NrJ   r*   r)   r   Z!MultiMarginLoss_weights_no_reducec                    s"   t j|  |  | ddS r   r~  rq   r   r"   r#   r7     s    z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                   S   ra  rb  r   r"   r"   r"   r#   r7     rR   rx   r   c                    s    t d |  j|  ddS )Nr  rm   r   rW  rQ   r   r"   r#   r7     r  TFr\  )r.   rw   r   r
  r   rY   r   rj   r"   r"   r   r#   &multimarginloss_weights_no_reduce_test  s    
r  c                 C   s^   dd }|| }t |tjr|gn|}t  || dW  d   S 1 s(w   Y  dS )zReference function for modules supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    c                 S   &   t | ttfrdd | D S | dS )Nc                 S      g | ]}| d qS r   	unsqueeze.0r0   r"   r"   r#   
<listcomp>/  rs   zDsingle_batch_reference_fn.<locals>.unsqueeze_inp.<locals>.<listcomp>r   
isinstancelisttupler  inpr"   r"   r#   unsqueeze_inp-     
z0single_batch_reference_fn.<locals>.unsqueeze_inpr   N)r  r.   Tensorr   squeeze)r/  
parametersmoduler  Zsingle_batch_inputr"   r"   r#   single_batch_reference_fn'  s   $r  c                  C   s  g t  t t t t t t t t t	 t
 t t t t t t t t t t t t t t t t t t t t t t  t! t" t# t$ t% t& t' t( t) t* t+ t, t- t. t/ t0 t1 t2dddddddt3j4dt2ddd	ddd
ddt3j4d	t2ddddddddt3j4d	t2ddddddddt3j4d	t2ddddddddt3j4d	t2ddddddddt3j4d	t2dddddddddt2ddd d dddt3j4d!t2d"d#d d$d%dddt3j4d&t2d'd(d d)ddddt3j4d&t2d*d+d d,ddddt3j4d&t2d-d.d d/ddddt3j4d&t2d0d1d d2ddddt3j4d&t2d3d4d d5dd6ddt3j4d7t2d3d8d9d:dd;ddt3j4d	t2d3d<d=d:dd>ddt3j4d	t2d?d@d dAddBddt3j4d7t2dCdDdEdFddddt3j4dG	t2dCdHdIdJddKdddt3j4dL
t2dCdMdNdJddOdddt3j4dL
t2dCdPdQdRdd>dddt3j4dL
t2dCdSdTdUdd;dddVt3j4dL
t2dCdDdEdWdddddXt2dYdZd d[d\ddddVt3j4d]	t2d^d_d d[d\dddVt3j4d`t2dadbd dcdddddt3j4d&t2dedfd dgdddddt3j4d&t2dhdid djdddddt3j4d&t2dkdldmddndddt3j4do	t2dkdpdqdrdd>dddt3j4dL
t2dkdsdtdrdd;dddt3j4dL
t2dudvd dwdxddddt3j4d]	t2dydzd d{d|ddt3j4d!t2d}d~d dd|ddt3j4d!t2ddd dd|ddt3j4d!t2ddd dd|ddt3j4d!t2ddd ddddt3j4d!t2ddddddddt3j4dG	t2dddddd;dddt3j4dL
t2dddddddddt3j4dL
t2dddddd
dddt3j4dL
t2dddddddddt3j4dL
t2dddddddddt2ddd ddddddt3j4d]	t2ddd ddddt3j4d!t2ddd ddddt3j4d!t2ddd dddddt3j4d&t2ddd dddddt3j4d&t2ddd dddddt3j4d&t2dddddddt3j4dt2dddddd>ddt3j4d	t2ddddt3j4dt2ddddt5dt3j4dt2ddddd dddt2ddddd dt3j4t6dǃdȍt2ddddd ddt3j4t6dǃdˍt2ddddd ddt3j4dЍt2ddddd ddt3j4dЍt2ddddd ddt3j4dЍt2ddddd ddt3j4dЍt2ddd ddd dt3j4dލt2ddd ddd dt3j4dލt2ddd ddd dt3j4dލt2ddd ddd dddt2dd ddd ddddt2ddddt3j4dt2ddddt3j4dt2t7t8j9dd ddddddt3j4dt2t7t8j9dd ddddd ddt2t7t8j9dd ddddddt3j4dt2t7t8j9d dddddddt3j4dt2t7t8j9dd ddd	d
dddt3j4dt2t7t8j9dd ddd	ddddt3j4dt2t7t8j9d dddd	ddddt3j4dt2t7t8j9dd ddd	d
ddddt2t7t8j9dd ddd	ddddt3j4dt2t7t8j9d dddd	ddddt3j4dt2t7t8j9dd ddddddt3j4dt2t7t8j9dd ddddddt3j4dt2t7t8j9dd ddddd dt3j4dt2t7t8j9d dddd!dd"dt3j4dt2t7t8j9dd dddd#d$ddt2t7t8j9dd d%dd	d&dd'dt3j4dt2t7t8j9dd d%dd	d&d#d(ddt2t7t8j9d)d d%dd	d*d+d,dt3j4dt2t7t8j9d dd%dd	d-dd.dt3j4dt2t7t8j9d d/d%dd	d0dd1dt3j4dt2t7t8j9d d2d%dd	d3dd4dt3j4dt2t7t8j9d)d d%dd	d5dd6dt3j4dt2t7t8j9d d2d%dd	d7dd8dt3j4dt2t7t8j9dd d9dd	d:dd;dt3j4dt2t7t8j9dd d9dd	d:d#d<ddt2t7t8j9d)d d9dd	d=d+d>dt3j4dt2t7t8j9d dd9dd	d?dd@dt3j4dt2t7t8j9d d/d9dd	dAddBdt3j4dt2t7t8j9d d2d9dd	dCddDdt3j4dt2t7t8j9d)d d9dd	dEddFdt3j4dt2t7t8j9d d2d9dd	dGddHdt3j4dt2t7t8j9dd dddIdJdKdt3j4dt2t7t8j9dd dddIdLdMddt2t7t8j9dNd dddOdPdQdt3j4dt2t7t8j9d ddddRdJdSdt3j4dt2t7t8j9dd dTdd	dUdJdVdt3j4dt2t7t8j9dd dTdd	dUdLdWddt2t7t8j9dXd dTdd	dYdZd[dt3j4dt2t7t8j9d d\dTdd	d]dd^d_dt3j4d`t2t7t8j9dXd dTdd	dadZdbdt3j4dt2t7t8j9d d\dTdd	dcdPddd_dt3j4d`t2t7t8j:dedfdgdhdidt3j4dt2t7t8j:djt3j;dkdldhdmddt3j4dnt2t7t8j:djdfdodpdqdt3j4dt2t7t8j:djdfdodrdsdt3j4dt2t7t8j:djt3j;dkdldrdtddt3j4dnt2t7t8j:dudfdvdwdxddt3j4dyt2t7t8j:dzdfd{dwd|ddt3j4dyt2t7t8j:dedfdgd}d~dddt2t7t8j<dedfddhddt3j4dt2t7t8j<djdfddpddt3j4dt2t7t8j<djdfddrddt3j4dt2t7t8j<dudfddwddt3j4dt2t7t8j<dzdfddwddt3j4dt2t7t8j<dudfdd}dddt2ddd ddddt3j4dt2ddd ddddt3j4dt2ddd dddt5dt3j4dt2ddd ddddt3j4dt2ddd ddddt3j4dt2ddd ddt5ddt3j4dt2dddd}ddt3j4dt2ddd t3j4dt2ddd dt3j4dt2ddd dt3j4dt2ddddd dt3j4dt2ddd t5dt3j4dt2dddddddddt3j4d
t2dddddt8j=fdddddt>
rBdndt3j4d	t2ddÐdĐdd ddddt3j4dƍ	t2dddddt8j=fdǐdd ddddt3j4dƍ	t2dɐddddddt8j?fdʐdd dddt>
rdndt3j4dƍ	t2dΐdϐdАdd dd dddt3j4dӍ	t2dԐdՐd֐dt5dt3j4d؍t2dِdڐdt3@ddgfdt5dt3j4d؍t2dg dݢddfdߐddddddd	} tAg dg dD ]\}}dD ]{}|dzkr&|dkr&qtBtCdj|dj }ddDtEtF| d }dd|  }dtBdd |D  }| Gt2d| dddzdzd|djdjd|f	d| d| d| d||d| dddt3j4d
 qqg d}i ddt3j4dddd2idddt3j4ddddt3j4dddt3j4ddd t3j4idd t3j4idd t3j4idd t3j4idd t3j4idd t3j4idd t3j4idd t3j4idd t3j4id	d t3j4id
d t3j4idd t3j4id t3j4id t3j4id t3j4id t3j4id t3j4id t3j4id t3j4id}	|D ]}
t2|
dt5ddd}|	H|
i }|I| | G| qS| S (  NConv1d)r+   rJ   rN   z!torch::nn::Conv1dOptions(4, 5, 3))rE   r+   r)   Tr8   )r9   r:   r;   r<   cudnnr>   r?   r@   )r+   rJ   rN   rE   z+torch::nn::Conv1dOptions(4, 5, 3).stride(2)stride)	r9   r:   r;   r<   r  rC   r>   r?   r@   )r+   rJ   rN   r,   r,   z6torch::nn::Conv1dOptions(4, 5, 3).stride(1).padding(1)Zpad1r   )r+   rJ   rJ   r,   rE   z6torch::nn::Conv1dOptions(4, 5, 5).stride(1).padding(2)Zpad2)r+   r+   rN   r,   r,   z6torch::nn::Conv1dOptions(4, 4, 3).stride(1).padding(1))r,   r+   r,   Z	pad1size1)r+   r+   rJ   r,   rE   z6torch::nn::Conv1dOptions(4, 4, 5).stride(1).padding(2)Z	pad2size1)r   r+   r)   Z
zero_batch)r9   r:   r;   r<   r  rC   r>   r?   ZConv1d_dilatedc                   S      t jdddddS )Nr+   rJ   rN   rE   kernel_sizedilationrh   r  r"   r"   r"   r#   r7     r{   z&get_new_module_tests.<locals>.<lambda>z-torch::nn::Conv1dOptions(4, 5, 3).dilation(2))r}   r~   r;   r<   r>   r?   r@   ZConv1d_groupsc                   S   r  )Nr+   rU   rN   rE   r  groupsr  r"   r"   r"   r#   r7     r{   z+torch::nn::Conv1dOptions(4, 6, 3).groups(2))rE   r+   rU   )r}   r~   r;   r<   r  r>   r?   r@   ZConv1d_pad_validc                   S   r  )Nr+   rJ   rN   validpaddingr  r"   r"   r"   r#   r7     r{   z8torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kValid)ZConv1d_pad_samec                   S   r  )Nr+   rJ   rN   samer  r  r"   r"   r"   r#   r7     r{   z7torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame)ZConv1d_pad_same2c                   S   s   t jdddddS )Nr+   rJ   r  r  r  r"   r"   r"   r#   r7     r{   z7torch::nn::Conv1dOptions(4, 5, 4).padding(torch::kSame)ZConv1d_pad_same_dilatedc                   S   s   t jddddddS )Nr+   rJ   r  rE   r  r  r  r"   r"   r"   r#   r7     rB   zCtorch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame).dilation(2)ConvTranspose1dc                   S   s   t jdddddddS )NrN   r+   rN   r,   r,   )r  r  r  output_paddingrh   r  r"   r"   r"   r#   r7     rs   zQtorch::nn::ConvTranspose1dOptions(3, 4, 3).stride(3).padding(1).output_padding(1))r,   rN      )r}   r~   r;   r  r<   r>   r?   r@   )rN   r+   rN   rE   r,   r,   r,   Fztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(false))r,   rN   rU   rA   )	rN   r+   rN   rE   r,   r,   r,   TrE   ztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(true).dilation(2)ZdilatedZConvTranspose1d_groupsc                	   S   s   t jddddddddS )	Nr+   rU   rN   r  r,   r  rE   )r  r  r  r  r  r"   r"   r"   r#   r7     r   ztorch::nn::ConvTranspose1dOptions(4, 6, 3)
                                    .stride(3).padding(1).output_padding(1).groups(2))rE   r+   r  Conv2d)rN   r+   rN   rE   z&torch::nn::Conv2dOptions(3, 4, {3, 2}))rE   rN   r  rJ   )	r9   r:   r;   r<   r  check_with_long_tensorr>   r?   r@   )rN   r+   rN   rN   rE   rE   z5torch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2})rT   Zstrided)
r9   r:   r;   r<   r  rC   r  r>   r?   r@   )rN   r+   r  r  r,   r,   zEtorch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2}).padding({1, 1})r  )rN   rE   r  r  r  r  zVtorch::nn::Conv2dOptions(3, 2, {3, 3}).stride({2, 2}).padding({1, 1}).dilation({2, 2}))rE   rN   r*   r*   )rN   r+   r  r,   r   r,   r,   Fztorch::nn::Conv2dOptions(3, 4, {3, 2})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))rE   rN   rU   rJ   gQ?)r   rN   r  rJ   )r9   r:   r;   r<   r  rC   r  r>   ZConv2d_groupsc                   S   r  Nr+   rU   r  rE   r  rh   r  r"   r"   r"   r#   r7   g  r{   z0torch::nn::Conv2dOptions(4, 6, {3, 2}).groups(2))rE   r+   rU   rJ   )	r}   r~   r;   r<   r  r  r>   r?   r@   ZConv2d_groups_thnnc                   S   r  r  r  r"   r"   r"   r#   r7   r  r{   )r}   r~   r;   r<   r  r>   r?   r@   ZConv2d_pad_validc                   S   r  )NrE   r+   rN   r+   r  r  r  r"   r"   r"   r#   r7   |  r{   z=torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kValid))rE   rE   rU   rJ   ZConv2d_pad_samec                   S   r  )NrE   r+   r  r  r  r  r"   r"   r"   r#   r7     r{   z<torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame)ZConv2d_pad_same_dilatedc                   S   s   t jddddddS )NrE   r+   r  r  r  r  r"   r"   r"   r#   r7     rB   zHtorch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame).dilation(2)ConvTranspose2d)rN   r+   rN   r  r,   r  ztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({3, 2}).padding(1).output_padding({1, 1}))r,   rN   r  rU   )	r9   r:   r;   r  r<   r  r>   r?   r@   )	rN   r+   rN   rE   rN   r,   r  r,   Fr  a`  torch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3})
                                    .padding(1)
                                    .output_padding({1, 1})
                                    .groups(1)
                                    .bias(false)
                                    .dilation({2, 2}))r,   rN   rU   r  )rN   r+   rN   r  r,   r  r,   Fztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3}).padding(1).output_padding({1, 1}).groups(1).bias(false)ZConvTranspose2d_groupsc                   S      t jdddddS )NrE   r+   r  r  )rh   r  r"   r"   r"   r#   r7     r{   z9torch::nn::ConvTranspose2dOptions(2, 4, {2, 3}).groups(2))r,   rE   r+   rJ   ZConv2d_depthwisec                   S   s   t jdddddS )Nr+   r  r  r  r"   r"   r"   r#   r7     r{   z0torch::nn::Conv2dOptions(4, 4, {3, 3}).groups(4))rE   r+   rU   rU   Z Conv2d_depthwise_with_multiplierc                   S   r  )Nr+   r*   r  r  r  r"   r"   r"   r#   r7     r{   z0torch::nn::Conv2dOptions(4, 8, {3, 3}).groups(4)ZConv2d_depthwise_stridedc                   S      t jddddddS )Nr+   r  r  )r  r  r  r"   r"   r"   r#   r7     rB   z?torch::nn::Conv2dOptions(4, 4, {3, 3}).stride({2, 2}).groups(4)ZConv2d_depthwise_paddedc                   S   r  )Nr+   r  r  )r  r  r  r"   r"   r"   r#   r7     rB   z@torch::nn::Conv2dOptions(4, 4, {3, 3}).padding({1, 1}).groups(4)ZConv2d_depthwise_dilatedc                   S   s   t jddddddS )Nr+   r  )r  r  r  r"   r"   r"   r#   r7     rB   zAtorch::nn::Conv2dOptions(4, 4, {2, 2}).dilation({2, 2}).groups(4))rE   r+   rJ   rJ   Conv3d)rE   rN   rE   rN   rE   z)torch::nn::Conv3dOptions(2, 3, {2, 3, 2}))r,   rE   r+   rJ   r+   g?)rE   rN   rE   rN   r+   r,   r   r,   r,   Fztorch::nn::Conv3dOptions(2, 3, {2, 3, 4})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))r,   rE   rN   r+   rJ   )rE   rN   )r,   r,   r,   r,   r   r,   r,   FZ1x1x1_no_biasF)rN   r+   rE   rE   z+torch::nn::Conv3dOptions(3, 4, 2).stride(2))rE   rN   rJ   rJ   rJ   )rN   r+   rE   rE   r,   z6torch::nn::Conv3dOptions(3, 4, 2).stride(2).padding(1)Zstride_padding)rN   r+   r  z)torch::nn::Conv3dOptions(3, 4, {2, 3, 4}))r   rN   rN   r+   rJ   )r9   r:   r;   r<   r  r  rC   r>   ZConv3d_groupsc                   S   r  )NrE   r+   rN   r  rh   r  r"   r"   r"   r#   r7   D  r{   z+torch::nn::Conv3dOptions(2, 4, 3).groups(2)ZConv3d_dilatedc                   S   s   t jdddddS )NrN   r+   rE   r  r  r"   r"   r"   r#   r7   O  r{   z-torch::nn::Conv3dOptions(3, 4, 2).dilation(2)ZConv3d_dilated_stridedc                   S   s   t jddddddS )NrN   r+   rE   )r  r  r  r  r"   r"   r"   r#   r7   X  rB   z7torch::nn::Conv3dOptions(3, 4, 2).dilation(2).stride(2)ZConv3d_pad_validc                   S   r  )NrN   r+   r  r  r  r  r"   r"   r"   r#   r7   a  r{   z@torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kValid))rE   rN   rU   rJ   r+   ZConv3d_pad_samec                   S   r  )NrN   r+   r  r  r  r  r"   r"   r"   r#   r7   k  r{   z?torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame)ZConv3d_pad_same_dilatedc                   S   s   t jddddddS )NrN   r+   r  r  rE   r  r  r"   r"   r"   r#   r7   u  rB   zKtorch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame).dilation(2)ZConvTranspose3dz2torch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2}))r9   r:   r;   r  r<   r>   r?   r@   )	rE   rN   r  r,   r   r   r,   T)rE   rE   rE   ztorch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2})
                                    .stride(1).padding(0).output_padding(0).groups(1).bias(true).dilation({2, 2, 2}))	r9   r:   r;   r  r<   rC   r>   r?   r@   ZReplicationPad3d))r,   rE   rN   rN   rE   r,   z6torch::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1}))rE   rN   rE   rE   rE   )r9   r:   r;   r<   r@   )rN   rE   rE   rE   Zno_batch_dim)r9   r:   r;   r<   r=   rC   r@   c                	   S   s   t jdddddt jddS )NrE   rN   Tr   requires_grad)r.   rw   
complex128r"   r"   r"   r#   r7     rR  complex)r9   r:   r;   r   	skip_halfrC   	Embedding)r+   rN   z!torch::nn::EmbeddingOptions(4, 3)c                   S      t jddt jddS NrE   rN   r   r+   r.   r  r   random_r"   r"   r"   r#   r7     r   z0https://github.com/pytorch/pytorch/issues/117971)r9   r:   r;   r   rV   r@   	decoratorc                   S       t jddt jddddS Nr,   i   r   r+   r  r.   r  r   r  r2   r"   r"   r"   r#   r7          Zdiscontiguous)r9   r:   r;   r   rV   rC   r@   r  EmbeddingBagz$torch::nn::EmbeddingBagOptions(4, 3)c                   S   r  r  r  r"   r"   r"   r#   r7     r   mean)r9   r:   r;   r   rV   rC   r@   c                   S   r  r  r  r"   r"   r"   r#   r7     r  )r+   rN   N       @Fry  ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum)c                   S   r  r  r  r"   r"   r"   r#   r7     r   ry  )r+   rN   Nr  Fmaxztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax)c                   S   r  r  r  r"   r"   r"   r#   r7     r   r  ZEmbeddingBag_mean_padding_idxc                   S   s   t jddddS )Nr+   rN   r,   Zpadding_idxrh   r  r"   r"   r"   r#   r7     r   z3torch::nn::EmbeddingBagOptions(4, 3).padding_idx(1)c                   S      t t dt dgS NrN   r.   stackrX   r"   r"   r"   r#   r7     rR  )r}   r~   r;   r   rV   r@   ZEmbeddingBag_sum_padding_idxc                	   S      t jddd dddddS )Nr+   rN   r  Fry  r,   r  r  r"   r"   r"   r#   r7     r   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum).padding_idx(1)c                   S   r  r  r  r"   r"   r"   r#   r7     rR  ZEmbeddingBag_max_padding_idxc                	   S   r  )Nr+   rN   r  Fr  r,   r  r  r"   r"   r"   r#   r7     r   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax).padding_idx(1)c                   S   r  r  r  r"   r"   r"   r#   r7     rR  ZEmbeddingBag_sparsec                   S   s   t jdddtjdS )Nr+   rN   T)sparser   )rh   r  r.   rY   r"   r"   r"   r#   r7      rB   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                   S      t dddS NrE   r,   r.   rX   repeatr"   r"   r"   r#   r7     r{   )r}   r~   r;   r   rV   has_sparse_gradientsc                   S   s   t jddtjddS )Nr+   rN   T)r   r  )rh   r  r.   rY   r"   r"   r"   r#   r7     rB   z_torch::nn::EmbeddingOptions(4, 3).sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                   S   r  r  r  r"   r"   r"   r#   r7   
  r{   ZEmbedding_sparse)r~   r;   r   r}   rV   r  ZPixelShuffler  z!torch::nn::PixelShuffleOptions(3))r,   	   r+   r+   ZPixelUnshufflez#torch::nn::PixelUnshuffleOptions(3))r,   r,      r  r  Znearest)rZ   scale_factormodezF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12})).scale_factor(std::nullopt).mode(torch::kNearest))r,   rE   r+   Zinterpolate_nearest_1d)r~   cpp_options_argsr<   r}   r   r@   )r   rE   r+   Zinterpolate_nearest_1d_zero_dim)r~   r  r<   r}   r   )r  )r,   rE   rN   Zinterpolate_nearest_tuple_1dg      @zF::InterpolateFuncOptions()
                                .size(std::nullopt).scale_factor(std::vector<double>({4.})).mode(torch::kNearest)Zinterpolate_nearest_scale_1dZlinear)rZ   r  r  Zalign_cornersa  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)Zinterpolate_linear_1d)r+   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)Zinterpolate_linear_tuple_1da  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(false)Zinterpolate_linear_scale_1dZinterpolate_linear_1d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(true)Z#interpolate_linear_1d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(true)Z)interpolate_linear_scale_1d_align_cornersrE   zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({2, 2}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r,      r,   r,   Z%interpolate_nearest_2d_launch_configszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r,   rE   r+   r+   Zinterpolate_nearest_2d)r     zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r,   rE   rN   r+   Zinterpolate_nearest_tuple_2dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kNearest)Zinterpolate_nearest_scale_2d)r   rE   r+   r+   Zinterpolate_nearest_2d_zero_dimZbilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false)Zinterpolate_bilinear_2dZ interpolate_bilinear_2d_zero_dim)r+   rU   a	  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false))r,   rE   rE   rN   Zinterpolate_bilinear_tuple_2da
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)Zinterpolate_bilinear_scale_2d)r  r  a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)Z*interpolate_bilinear_scale_tuple_shared_2d)r        ?a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)Z*interpolate_bilinear_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(true)Z+interpolate_bilinear_tuple_2d_align_cornersa	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(true)Z8interpolate_bilinear_scale_tuple_skewed_2d_align_cornersZbicubica
  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)Zinterpolate_bicubic_2dZinterpolate_bicubic_2d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)Zinterpolate_bicubic_tuple_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBicubic)
                                .align_corners(false)Zinterpolate_bicubic_scale_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBicubic)
                                .align_corners(false)Z)interpolate_bicubic_scale_tuple_shared_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(false)Z)interpolate_bicubic_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(true)Z*interpolate_bicubic_tuple_2d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(true)Z7interpolate_bicubic_scale_tuple_skewed_2d_align_cornerszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r,   rE   r+   r+   r+   Zinterpolate_nearest_3d)r   rE   r+   r+   r+   Zinterpolate_nearest_3d_zero_dim)r  r  r  zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r,   rE   rN   r+   r+   Zinterpolate_nearest_tuple_3dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4., 4.}))
                                .mode(torch::kNearest)Zinterpolate_nearest_scale_3dZ	trilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false)Zinterpolate_trilinear_3dZ!interpolate_trilinear_3d_zero_dim)r+   rU   rU   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false))r,   rE   rE   rN   rN   Zinterpolate_trilinear_tuple_3dg      @a  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(false)Zinterpolate_trilinear_scale_3dr   )r~   r  r<   r}   r   r   r@   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(true)Z,interpolate_trilinear_tuple_3d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(true)Z,interpolate_trilinear_scale_3d_align_cornersr-   rv  zF::SoftmaxFuncOptions(-1))rE   r  Zsoftmax_lastdimr,   )rw  r   z/F::SoftmaxFuncOptions(1).dtype(torch::kFloat64)Zsoftmax_lastdim_dtype)r~   r  r<   r}   r   rF   r@   zF::SoftmaxFuncOptions(1))rE   r  rE   rE   Zsoftmax_spatial_special)rE   rE   r+   r+   Zsoftmax_spatialZsoftmax_spatial_dtyper   zF::SoftmaxFuncOptions(0)rM   Zsoftmax_functional_dim0)r~   r  r<   r}   rF   r   r@   rN   zF::SoftmaxFuncOptions(3)Zsoftmax_functional_dim3r"   Zsoftmax_functional_scalar)r~   r  r<   r}   rF   r   zF::LogSoftmaxFuncOptions(-1)Zlog_softmax_lastdimzF::LogSoftmaxFuncOptions(1)Zlog_softmax_spatial_specialZlog_softmax_spatialzF::LogSoftmaxFuncOptions(0)Zlog_softmax_dim0zF::LogSoftmaxFuncOptions(3)Zlog_softmax_dim3Zlog_softmax_scalarUnfoldc                   S      t ddddS )Nr  r  r   r   rh   r  r"   r"   r"   r#   r7   q	  r   zPtorch::nn::UnfoldOptions({2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rE   r+   rN   rN   )r}   r~   r;   r<   rV   rF   r@   Foldc                   S      t dddddS Nr  r  r  r  rh   r  r"   r"   r"   r#   r7   z	  r{   zVtorch::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rE   r  r+   ZFold_no_batch_dim_inputc                   S   r  r  r  r"   r"   r"   r#   r7   	  r{   )r  r+   )r}   r~   r;   r<   rV   refrF   r@   ZUnfold_int_inputc                   S   r  )NrE   r,   r   r  r"   r"   r"   r#   r7   	  r   z<torch::nn::UnfoldOptions(2).dilation(1).padding(0).stride(1)ZFold_int_inputc                   S   r  NrN   rE   r,   r   r  r"   r"   r"   r#   r7   	  r{   z=torch::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)ZFold_no_batch_dim_int_inputc                   S   r  r  r  r"   r"   r"   r#   r7   	  r{   )r}   r~   r;   r<   r  rV   rF   r@   rD   rG   rI   Zwith_up_down_scalarrK   ZPairwiseDistancec                   S      t ddt ddfS Nr)   r*   r   r"   r"   r"   r#   r7   	  r   )r9   r   r@   c                   S   s   t ddt ddfS )Nr)   r,   r*   r   r"   r"   r"   r#   r7   	  r   Zbroadcast_lhs)r9   r   rC   r@   c                   S   s   t ddt ddfS )Nr)   r*   r,   r   r"   r"   r"   r#   r7   	  r   Zbroadcast_rhs)g      ?r   TzDtorch::nn::PairwiseDistanceOptions().p(1.5).eps(1e-05).keepdim(true)c                   S   r  r  r   r"   r"   r"   r#   r7   	  r   Zwith_non_default_args)r9   r:   r;   r   rC   r@   c                   S      t dt dfS )Nr*   r   r"   r"   r"   r#   r7   	  rB   )r9   r   r=   rC   r@   ZTransformerEncoderLayer)r+   rE   r          ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(16)
                                    .dropout(0.0)r  Zrelu_activationrH   )
r9   r:   r;   r<   rC   r>   r?   rW   rV   r@   r+   r*   r   ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)Zgelu_activationg{Gz?)	r9   r:   r;   r<   rV   rC   r>   r?   r@   ZTransformerDecoderLayer)r+   rE   r*   r   ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)c                   S      t dddt dddfS NrN   r+   rE   rv   r"   r"   r"   r#   r7   	  r  )	r9   r:   r;   r   rV   rC   r>   r?   r@   ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)c                   S   r  r  rv   r"   r"   r"   r#   r7   
  r  ZTransformera  torch::nn::TransformerOptions()
                                    .d_model(4)
                                    .nhead(2)
                                    .num_encoder_layers(2)
                                    .num_decoder_layers(2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kReLU)c                   S   s&   t dddt dddt ddfS r  rv   r"   r"   r"   r#   r7   
  r   Zmultilayer_codergQ?r(   )rN   rJ   ztorch::nn::LinearOptions(3, 5)c                   S   r   r  rv   r"   r"   r"   r#   r7    
  r   c                 S   s*   t | dd|d  d|d  S )Nr,   r-   r   )r.   r/   r1   r0   r3   r"   r"   r#   r7   !
  r   )	r9   r:   r;   r   r=   rC   r>   r?   r@   rL   z5torch::nn::FlattenOptions().start_dim(-3).end_dim(-1))r-   )rN   r+   rJ   )r9   r;   r:   r<   r=   rC   r@   Z	Unflattenz'torch::nn::UnflattenOptions(-2, {2, 2})Z	LayerNorm)8   r  r  r   zMtorch::nn::LayerNormOptions({56, 56, 56}).eps(1e-5).elementwise_affine(false))r+   r  r  r  Z3d_no_affine_large_feature)	r9   r:   r;   r<   r  Z
check_evalgradcheck_fast_mode
check_halfrC   )reflectZcircularZ	replicaterZ  )ztorch::kReflectztorch::kCircularztorch::kReplicateztorch::kZerosr  {z, }r  r  c                 s   s    | ]}|d  V  qdS r,   Nr"   r  r5   r"   r"   r#   	<genexpr>V
      z'get_new_module_tests.<locals>.<genexpr>ZConvdztorch::nn::Convz~dOptions(2, 3, 3)
                                            .stride(2)
                                            .padding(z)
                                            .dilation(1)
                                            .groups(1)
                                            .bias(true)
                                            .padding_mode()Z_stride2_pad2)
r9   r:   r;   r<   output_sizer  rC   r>   r?   r@   )ELU
HardshrinkHardsigmoidHardtanh	Hardswish	LeakyReLU
LogSigmoidPReLUReLUReLU6rD   SELUCELUGELUGLUr   SiLUMishSoftplus
SoftshrinkSoftsignTanh
Tanhshrink	Thresholdr  )r  )r:   r@   r'  r:   r  )rV   Z	check_jitr@   r  )rF   r@   r  r@   r  r  r  r  r  r  r!  r  r  r  r  )r   r   r"  r#  r$  r%  r&  )r9   r<   r=   rC   test_cpp_api_parity)Jr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r(  r.  r0  r6  r9  r8  r;  r=  r<  rB  rD  rJ  rK  rO  r[  r]  rd  re  rl  rq  rs  r{  r|  r  r  r  r  r  r  r   r.   rY   r  r   rj   ro   ZinterpolateZsoftmaxfloat64log_softmaxZgelur   ZreluSizezipr  rangejoinmapr  appendgetupdate)Znew_module_testsZpadding_modeZcpp_padding_moder  r  Zcpp_paddingr<   r  Znon_linear_activations_no_batchZ!non_linear_activations_extra_infoZnon_linear_activationZactivation_test_info
extra_infor"   r"   r#   get_new_module_tests8  s  	
 !"#$%&'()*+,-./0123=HS^it~       &  0  :  D  P  \  g  r  ~           $    .    9    C    M    W    a    m                     !      *      3      <      E      P      ]      j      v                               !        *        4        >        H        R        ^        e        n        v                 
                              &          0          8          A          J          S          [          b          i          r          z                                               %            1            <            H            T            _            j            u                                                       "              /              <              I              V              b              o              {                                                                -                :                F                S                ^                h                s                ~                                   
                  #                  1                  >                  L                  T                  ]                  e                  m                  v                                     	                                                            !                    )                    1                    9                    @                    I                    R                    \                    e                    n                    x                                                                                                                                   "                      3                      A                      N                      \                      n                      y                                                                       
 

	









r4  r  c                 C   sr   |rt |||   }n|| |   }|dkr| S |dkr$| S |dkr7| dkr7| |d S |S )Nr  ry  Z	batchmeanr   )r.   rz   r   r  ry  rw  rZ   )r/  r   r   r   r!   r"   r"   r#   kldivloss_reference
  s   r5  c                 C   s   |   dksJ | d}| d}|f|  dd   }t|| }|d u r1t|| }d}	tdd |D  D ]+}
||
 }||krHdn||  }t|
}|	d| | t
|  | ||
< |	|7 }	q<|dkrr| |	 S |d	krz| S |S )
NrN   r   r,   rE   c                 S   s   g | ]}t |qS r"   )r-  )r  rZ   r"   r"   r#   r  
  rB   z'nlllossNd_reference.<locals>.<listcomp>r   r  ry  )rw  rZ   r.   rZ  rp   onesr   itemr  insertr  ry  )r/  r   r%   r  r   NCZout_sizeoutputZtotal_weighttupZt_nxnormZinput_indexr"   r"   r#   nlllossNd_reference
  s(   


r?  r   c                 C   s   |   dksJ t| d} | d}|d u rt|| }|jd|gdd | jdd  D R  }|dkrG|dks=J |d|  ||  }| | | jdd }|dkrZ|	 S |d	krb| S |S )
NrE   r,   c                 s       | ]}d V  qdS r  r"   r  r6   r"   r"   r#   r  
      z;cross_entropy_loss_prob_target_reference.<locals>.<genexpr>r   r  rv  r  ry  )
rw  r.   r*  rZ   r7  rp   r1   shapery  r  )r/  r   r%   r   label_smoothingr;  r<  r"   r"   r#   (cross_entropy_loss_prob_target_reference
  s   
(rE  c                 C   s4  t | d}tj|||||d}|dkr|S d|  k r!dks$J  J t | d} | d}|d urI| |jd|gdd | jdd  D R   } t | d }	||k}
|	|
d |dkr|d urwt |	|	d	|
|
    }nt |	
|
 }n|d
krt |	}n|	}d| | |||   S )Nr,   r  r   r  c                 s   r@  r  r"   rA  r"   r"   r#   r  
  rB  z>cross_entropy_loss_indices_target_reference.<locals>.<genexpr>rE   r  r   ry  )r.   r*  ro   r   rZ   r1   rC  ry  Zmasked_fill_gatherZmasked_selectZlogical_notrP   r  )r/  r   r%   r  r   rD  Zlog_softmax_inputZnlllossr;  Zsmooth_lossZignore_maskretr"   r"   r#   +cross_entropy_loss_indices_target_reference
  s4   
,*rH  c                 C   s2   | j |j krt| ||||dS t| |||||dS )N)r%   r   rD  )r%   r   r  rD  )rC  rE  rH  )r/  r   r%   r  r   rD  r"   r"   r#   cross_entropy_loss_reference  s   rI  c           	         sf   dd  fddt | |D }t | \}}| |}|dkr)t|t| S |dkr1t|S |S )Nc                 S   s6   ||krdS |d u rdn|| }| |  | }||fS )Nr  r,   r"   )r/  r   r%   r  r>  r!   r"   r"   r#   nll_loss_helper  s
   z*nllloss_reference.<locals>.nll_loss_helperc                    s   g | ]\}}|| qS r"   r"   )r  r4   r0   r  rJ  r%   r"   r#   r    s    z%nllloss_reference.<locals>.<listcomp>r  ry  )r,  Z
new_tensorry  )	r/  r   r%   r  r   Zlosses_and_weightslossesr&   Zlosses_tensorr"   rK  r#   nllloss_reference  s   
rM  r  c                 C   s~   | |   }||k|}||k |}|dkr|}n||d|   |d |d  |  }|dkr5| S |dkr=| S |S )Nr   rE  rE   r  ry  )r   rp   r  ry  )r/  r   r   rG  abs_diffZge_beta_maskZlt_beta_maskr<  r"   r"   r#   smoothl1loss_reference+  s   $rO  c                 C   sd   | |   }||k}||k }|| |d|   |d |d   }|dkr(| S |dkr0| S |S )NrE  rE   r  ry  )r   r  ry  )r/  r   r   rM  rN  Zge_delta_maskZlt_delta_maskr<  r"   r"   r#   huberloss_reference;  s   $rP  c                 C   sp   g }|D ]}|dk r n| | qd}|D ]}tdt| D ]}||vr4|tdd| |  | |  7 }qq|S )Nr   r,   )r0  r-  lenr  )r/  r   targetsZtarget_indexry  r4   r"   r"   r#   _multilabelmarginloss_referenceG  s   rS  c                 C   s   |   }|   dk r8|  dk sJ |   dkr| dn| dd} |  dkr0|dn|dd}| d}| d}| | }td|D ]}t| | || ||< qN|dkrf| | S |dkrp| | S |dk rz|	 | S || S NrE   r,   r   r  ry  )
rw  r  rZ   newzero_r-  rS  r  ry  r  )r/  r   r   Z	input_dimnrw  r<  r4   r"   r"   r#   multilabelmarginloss_referenceW  s"   &&

rX  c                 C   sL   ||  j dd| }t|dk| |}|dkr| S |dkr$| S |S )Nr   r_  r,   r  ry  )rc  rp   r.   wherer  ry  )r/  r   rp  r   Zmargin_clampr<  r"   r"   r#   hingeembeddingloss_referenceq  s   rZ  c                 C   s:   d|  |     }|dkr| S |dkr| S |S )Nr,   r  ry  )rz   r   r  ry  )r/  r   r   r<  r"   r"   r#   softmarginloss_reference|  s   r[  c                 C   sj   |d u r|  t| d}d}tdt| D ]}||kr2||| td|| |  | |  |  7 }q|S )Nr,   r   )rU  rQ  Zfill_r-  r  )r/  Z
target_idxr5   rp  r%   r<  r4   r"   r"   r#   _multimarginloss_reference  s   *r\  r,   c                 C   s   |   dk r|   dkr| dn| dd} |  }|  dkr(|d}| d}| d}| |}	td|D ]}
t| |
 ||
 ||||	|
< q<|dkrW|	 | S |dkra|	 | S |dkrl|	d| S |	| S rT  )	rw  r  rZ   rU  r-  r\  r  ry  r  )r/  r   r5   rp  r%   r   Z
target_dimrW  rw  r<  xr"   r"   r#   multimarginloss_reference  s"   &



r^  c                 C   sZ   dd }t |dkd|| | || || jdd}|dkr#| S |dkr+| S |S )Nc                 S   sv   |  | d}td| dD ](}| | ||   | | | |   d || ||   d  d  ||< q|S )Nr   g-q=rE  )rU  rZ   r-  ry  )abcosr4   r"   r"   r#   _cos  s   Nz+cosineembeddingloss_reference.<locals>._cosr,   r   r_  r  ry  )r.   rY  rc  r  ry  )input1input2r   rp  r   rb  r<  r"   r"   r#   cosineembeddingloss_reference  s   .re  rE   ư>c                 C   sz   t | |||}t | |||}	|r t ||||}
t |	|
}	t j|| |	 dd}|dkr3| S |dkr;| S |S )Nr   r_  r  ry  )r.   Zpairwise_distancer`  rc  r  ry  )anchorZpositivenegativerp  r5   epsZswapr   Zd_pZd_nZd_sr<  r"   r"   r#   tripletmarginloss_reference  s   rj  c                 C   s>   | | |  | j dd}|dkr| S |dkr| S |S )Nr   r_  r  ry  )rc  r  ry  )rc  rd  r   rp  r   r<  r"   r"   r#   marginrankingloss_reference  s   rk  c                 C   s  t j|t jd}t j|t jd}| j}|  } | }|d}g }t| dD ]}	||	  }
||	  }||	  }|	d| d f|}|
 dkr\||	d |f |dd d< n||| | |dd d< | d |
|	f  }| |d d f}|d|f |d< |d|d f |d< |d d |dd  k}td|
D ]2}| }|dd   |d d 7  < |dd   t ||d d |d7  < |||f | }q||dd    d    q)t |d}|dkr||j|j|jd  }n	|d	kr| }||}|S )
Nr   r   r,   rE   r  r-   r  r   devicery  )r.   Z	as_tensorr   r   rY   Zcumsumr-  rZ   r8  Znew_fullrw  rz   Z	new_zerosclonerY  r0  ry  r   catr   rm  r  )Z	log_probsrR  Zinput_lengthsZtarget_lengthsblankr   dtZcum_target_lengthsrL  r4   Zinput_lengthZtarget_lengthZcum_target_lengthZtargets_primeZprobsalphaZ
mask_thirdr0   Z
alpha_nextr<  r"   r"   r#   ctcloss_reference  sB   
,"

rs  )r   )r   r   r  r4  rA  rN  rV  rh  rr  r  CosineEmbeddingLossTripletMarginLossMarginRankingLossZCTCLossZCrossEntropyLossr  r   c                     s`   | d }dd  fdd  fdd| dd D }|| }t |}|d	kr.|d
S |S )zReference function for criterion supporting no batch dimensions.

    The criterion is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r-   c                 S   r  )Nc                 S   r  r  r  r  r"   r"   r#   r    rs   zNsingle_batch_reference_criterion_fn.<locals>.unsqueeze_inp.<locals>.<listcomp>r   r  r  r"   r"   r#   r    r  z:single_batch_reference_criterion_fn.<locals>.unsqueeze_inpc                    s<   g }t | ttfr| D ]	}| | q|S ||  |S r]   )r  r  r  extendr0  )Zxsr!   r]  )rP   r"   r#   rP     s   
z4single_batch_reference_criterion_fn.<locals>.flattenc                       g | ]} |qS r"   r"   )r  r/  )r  r"   r#   r  #  rB   z7single_batch_reference_criterion_fn.<locals>.<listcomp>Nrm   r   )r$   r  )r_   	criterionZsingle_batch_input_argsr<  r   r"   )rP   r  r#   #single_batch_reference_criterion_fn  s   	
rz  )ZL1LossZMSELossZPoissonNLLLossrN  rA  )rm   r  ry  Z_no_batch_dim_)namec                 G      t t| tdS Nrn   r   rh   r   r{  r_   r"   r"   r#   r7   6  r   r  )r}   r~   r<   Ztarget_sizer=   r(  r@   ZKLDivLoss_no_batch_dim_c                   C   s   t jtdS r}  )rh   r   r   r"   r"   r"   r#   r7   C  rR   c                   C   r   Nr  r   r"   r"   r"   r#   r7   D  r   c                   C   r   r  rv   r"   r"   r"   r#   r7   E  r   )r}   r~   r   	target_fnr=   r(  r@   BCELossc                   C   s   t t jdt jdS Nr  r   )r.   r   r   rY   r"   r"   r"   r#   r7   R  rs   c                   C   s   t jdt jddt jS )Nr  r   r   )r.   r   rY   r   r   r"   r"   r"   r#   r7   S  r   BCEWithLogitsLossc                   C      t jdt jdS r  r.   r   rY   r"   r"   r"   r#   r7   U  r   rh  c                   C   r  r  r  r"   r"   r"   r#   r7   V  r   c                   C      t g dd S N)r-   r,   r,   rN   r.   tensorr"   r"   r"   r#   r7   V  r{   rV  c                   C   r  )Nr+   r   r  r"   r"   r"   r#   r7   W  r   c                   C   s   t g dS )N)rN   r   r-   r,   r  r"   r"   r"   r#   r7   W  r   rr  c                   C   r  r  r  r"   r"   r"   r#   r7   X  r   c                   C   r  r  r  r"   r"   r"   r#   r7   X  r{   r  c                   C   s   t jtjdtjdddS )NrN   r   r   rv  )ro   r*  r.   r   rY   r"   r"   r"   r#   r7   Y  rR  c                   C   r   rO   r  r"   r"   r"   r#   r7   Y  r   rt  c                   C       t jdt jdt jdt jdfS r  r  r"   r"   r"   r#   r7   \  r  c                   C   r  )Nr,   r   )r.   r  rY   r"   r"   r"   r#   r7   ]  r   rv  c                   C   r  r   r   r"   r"   r"   r#   r7   `  rB   c                   C   r   r   )r.   r   signr"   r"   r"   r#   r7   `  r   ru  c                   C   r  r  r  r"   r"   r"   r#   r7   d  r  c                   C   r  r  r  r"   r"   r"   r#   r7   e  r   ZMultiLabelSoftMarginLossc                   C   r  r  r  r"   r"   r"   r#   r7   g  r   c                   C   r   )Nr  r   r"   r"   r"   r#   r7   g  r   rV   ,classification_criterion_no_batch_extra_info)r  r  rh  r  rr  c                 G   r|  r}  r~  r  r"   r"   r#   r7   y  r   c                 C      |  S r]   r"   fr"   r"   r#   r7   z      c                 C   r  r]   r"   r  r"   r"   r#   r7   {  r  )r}   r~   r   r  r=   r(  Z
has_parityc                   @   s   e Zd Zedd Zedejdeeej	 eej	 f fddZ
edejddfdd	Ze	
ddejdedejdeejeej f def
ddZdd Zdd Zdd ZddefddZddefddZd defddZdS )!
NNTestCasec                 O      t r]   NotImplementedErrorr^   r_   rb   r"   r"   r#   _forward     zNNTestCase._forwardr  returnc                 C   r  r]   r  r^   r  r"   r"   r#   _get_parameters  r  zNNTestCase._get_parametersNc                 C   r  r]   r  r  r"   r"   r#   _zero_grad_parameters  r  z NNTestCase._zero_grad_parametersFr/  r<  grad_outputcreate_graphc                 C   r  r]   r  )r^   r  r/  r<  r  r  r"   r"   r#   	_backward  s   zNNTestCase._backwardc                    sP   t |trt fdd|D S t |tr  fdd|D S t|  S )Nc                 3   s    | ]	} | V  qd S r]   	_jacobianr  elemnum_outr^   r"   r#   r        z'NNTestCase._jacobian.<locals>.<genexpr>c                    s   g | ]} | qS r"   r  r  r  r"   r#   r    r   z(NNTestCase._jacobian.<locals>.<listcomp>)r  r  r  r.   rZ  nelement)r^   r/  r  r"   r  r#   r    s
   

zNNTestCase._jacobianc                    s@   t |tjr|jr| dS |dS t fdd|D S )Nr-   c                 3       | ]}  |V  qd S r]   )_flatten_tensors)r  r_  r^   r"   r#   r        z.NNTestCase._flatten_tensors.<locals>.<genexpr>)r  r.   r  Z	is_sparseZto_denser1   r  )r^   r]  r"   r  r#   r    s
   
zNNTestCase._flatten_tensorsc                 C   sT   t |tjr|jr|jd ur|j  |j  d S d S d S |D ]}| | q d S r]   )r  r.   r  r  gradrV  Zdetach__zero_grad_input)r^   r/  r4   r"   r"   r#   r    s   
zNNTestCase._zero_grad_inputTc                 C   sP  |  ||}| }|r| ||}tt|}|r.tdd | |d D }	t|	|}
t	|D ]c}| |\}}dd t
||D }t|}|d}d||< |rZ| | |ra| | | ||||}|rt
|t|D ]\}}| d|d d |f< qr|rt| |d|
d d |f< q2d}|r||f7 }|r||
f7 }|S )	Nc                 s       | ]}|  V  qd S r]   )numelr  r"   r"   r#   r    r  z2NNTestCase._analytical_jacobian.<locals>.<genexpr>r   c                 S   s&   g | ]\}}|d u rt |n|qS r]   )r.   
zeros_like)r  r5   r  r"   r"   r#   r    r   z3NNTestCase._analytical_jacobian.<locals>.<listcomp>r-   r,   r"   )r  r  r  r  r   ry  r  r.   rZ  r-  r,  r  r1   r  r  r  
contiguousro  r  )r^   r  r/  jacobian_inputjacobian_parametersr<  r  Zjacobian_inpZflat_jacobian_inputZ	num_paramZjacobian_paramr4   paramd_paramZd_outZ
flat_d_outd_inputZ
jacobian_xZd_xresr"   r"   r#   _analytical_jacobian  s>   





zNNTestCase._analytical_jacobianc                    s    fdd}d}|r|t ||ddf7 }|r? \}}g }	|D ]}
t |||
dd}|	|d d  q"|t|	df7 }|S )Nc                     s     |  S r]   )r  detach)r/  r  r^   r"   r#   fw  r  z*NNTestCase._numerical_jacobian.<locals>.fwr"   rf  )ri  )r   ri  r   )r   r  r0  r.   ro  )r^   r  r/  r  r  r  r  r  r6   Zto_catr5   Zjacobianr"   r  r#   _numerical_jacobian  s   zNNTestCase._numerical_jacobianc                 C   s   t | |d }| ||||}| ||||}tt|}tt|}g }	t||D ]\}
}|
 dkrD|	|
j	|dd
   q,t|	dkrU| t|	t d S d S )Nr   r-   )rr  )boolr  r  r  r  r   r,  r  r0  r"  r   r  rQ  ZassertLessEqual	PRECISION)r^   r  r/  r  r  Z
analyticalZ	numericalZanalytical_tZnumerical_tZdifferencesr_  rW  r"   r"   r#   check_jacobian  s   zNNTestCase.check_jacobianF)TTT)rd   re   rf   r   r  rh   ri   r  r  	Parameterr  r  r   r.   r  r   r   r  r  r  r  r  r  r  r  r"   r"   r"   r#   r    s0    
*
		*r  c                   @   s`   e Zd Zh dZdddZdd Zdd	 Zed
d Zedd Z	dd Z
dddZdd ZdS )TestBase>   r:   r/  
extra_args Nc                 K   s~   || _ || _|| _|| _| jD ]'}||vr6|d |vr6|d |vr6|dv r*d||< qt|   d| dq|| _i | _d S )N_fn_size>   r:   r  r"   z
: Specify z5 by a value, a function to generate it, or it's size!)	rC   r}   r~   r=   _required_arg_names
ValueErrorget_name_extra_kwargs
_arg_cache)r^   r~   rC   r=   r}   rb   r{  r"   r"   r#   __init__  s   
 

zTestBase.__init__c                 C   s8   | j d ur
d| j  S d| jj }| jr|d| j 7 }|S )NZtest_r6   )r}   r~   rd   rC   )r^   Z	test_namer"   r"   r#   r    s   

zTestBase.get_namec                    s6   t |tjr|S t|rt| fdd|D S |S )Nc                 3   r  r]   )_unpack)r  vr  r"   r#   r    r  z#TestBase._unpack.<locals>.<genexpr>)r  r.   r  r   type)r^   valuer"   r  r#   r    s
   zTestBase._unpackc                 C      |  ddS )Nr:   T_get_argr  r"   r"   r#   r:   #     zTestBase.constructor_argsc                 C   r  )Nr  Tr  r  r"   r"   r#   r  '  r  zTestBase.extra_argsc              
      s   || j v sJ || jvrX|d }|d }|| jv r"| j| | j|< n6|| jv r1| j|  | j|< n'|| jv sHJ d| d| d| d|    fdd  | j| | j|< |rb| | j| S | j| S )	Nr  r  z	Missing `z`, `z` or `z` for c                    s:   t | tr fdd| D S t | tjr|  S t| S )Nc                    rx  r"   r"   )r  smap_tensor_sizesr"   r#   r  <  rB   z?TestBase._get_arg.<locals>.map_tensor_sizes.<locals>.<listcomp>)r  r  r.   r  rY   r   )sizesr  r"   r#   r  :  s
   

z+TestBase._get_arg.<locals>.map_tensor_sizes)r  r  r  r  r  )r^   r{  unpackfn_nameZ	size_namer"   r  r#   r  +  s   


zTestBase._get_argTc                 C   s   |  d|S )Nr/  r  )r^   r  r"   r"   r#   
_get_inputF     zTestBase._get_inputc                 C   r  r]   r  )r^   	test_caser"   r"   r#   __call__I  s   zTestBase.__call__)r  NNr  )rd   re   rf   r  r  r  r  propertyr:   r  r  r  r  r"   r"   r"   r#   r     s    
	


r  c                	       s\   e Zd ZededejdedefddZ fddZd	d
 Z	dd Z
dd Zdd Z  ZS )
ModuleTestr  r  r/  r  c                 C   r  r]   r  )r^   r  r  r/  r"   r"   r#   _do_testO  r  zModuleTest._do_testc                    s   t  j|i | |dd| _|dd| _|dd| _|dd| _|dd| _|dd	| _|d
d| _	|dd | _
| j
d u rMt | _
d S d S )Nr  TrF   r   rV   !FIXME_no_cuda_gradgrad_comparisonFr   g-C6*?check_forward_onlyr@   )superr  r1  r  should_test_cudashould_test_picklerV   r  r   r  r@   r.   get_default_dtyper  	__class__r"   r#   r  S  s   

zModuleTest.__init__c           
   	   C   s@  t | j | j| j }|  }| jd ur7|||}t|}t|}| |||d |}|j	||dd | j
rC	 W d    d S | ||| | jrt .}||| t|| |d tj|dd}	|	|||||	| W d    n1 sw   Y  | ||| W d    d S 1 sw   Y  d S )Nr   F)exact_dtype)Zweights_only)r   r@   r~   r:   r  r=   r  r   r  assertEqualr  test_noncontigr  tempfileTemporaryFiler.   saveseekloadr  )
r^   r  r  r/  outZ	ref_inputZ
ref_moduleexpected_outr  Zmodule_copyr"   r"   r#   r  a  s.   


"zModuleTest.__call__c                    s   t |tr fdd|D S t |trt fdd|D S |}| }|}t|D ]}||dkr9|d } nq*tt||g|	|d
 }| dks]| dks]| r]J |j|_|S )Nc                    s   g | ]}  |qS r"   noncontiguizer  or  r"   r#   r  ~  rs   z,ModuleTest.noncontiguize.<locals>.<listcomp>c                 3   r  r]   r  r  r  r"   r#   r    r  z+ModuleTest.noncontiguize.<locals>.<genexpr>r,   r   )r  r  r  rw  r-  rZ   r.   r  Z
empty_likeselectr  r  Zis_contiguousr  )r^   objr  ndimrw  r  Z	noncontigr"   r  r#   r  |  s    

"$zModuleTest.noncontiguizec              	   C   s  t |tjr| dkrd S tdd |D rd S || || t 7 |||}t	|ddr7|d }|
|j }| }t|||||}t||d }W d    n1 s`w   Y  | |}| |}	tddd	D ]a\}
}|
r}|n|}t|r|n|	}|| || t : |||}t	|ddr|d }|||||}||| |j||d
dd |||d | W d    n1 sw   Y  qud S )Nr   c                 s   s(    | ]}t |tjr| d kV  qdS )r   N)r  r.   r  rw  )r  r4   r"   r"   r#   r    s   & z,ModuleTest.test_noncontig.<locals>.<genexpr>return_indicesFr,   TFrE   )r  g-C6?atolrtol)r  r.   r  rw  anyr  r  r   r  r   rU  rC  normal_rn  r   r  r  r  r   r  )r^   r  r  r/  r<  r  r  r  Znc_inputZnc_grad_outputZcontig_iZcontig_gr4   gor  r  r"   r"   r#   r    sB   


	


zModuleTest.test_noncontigc              	   C   s  t r| js
tdt| j |  }tjtj	i}t
|tr"|n|f}tdd |D }t||d}| j| j }| j| j 	  }||}	||}
t|	d |
d D ]
\}}|j| qW|| || || || |||}|||}t|ddr|d }|d }|j||| jddd td	D ]@}|  }||}|||||}|||||}|j||| jddd t|	d
 |
d
 D ]\}}|j||| jdd qq| j r| j!s|| }|| }t|ddr|d }|d }tj"|dd}||# }d|_$tj%j&||t|'  |dd}tj%j&||t|'  |dd}t||D ]\}}|j||| jddd q4|rg|( ) t(dd |D  }|( ) t(dd |D  }n|( t(dd |D  }|( t(dd |D  }tj%j&|||f t|'  dd}tj%j&|||f t|'  dd}|j||| jddd t||D ]\}}|j||| jddd q| *||| W d    d S 1 sw   Y  d S )NExcluded from CUDA testsc                 s   s$    | ]}t |tjo|jjV  qd S r]   )r  r.   r  r   
is_complexr  r"   r"   r#   r    s   " z'ModuleTest.test_cuda.<locals>.<genexpr>)type_mapr   r  Fr  r   r  rJ   r,   r  T)r  )r  c                 s       | ]	}|   V  qd S r]   ry  r   r  r]  r"   r"   r#   r    r  c                 s   r  r]   r	  r
  r"   r"   r#   r     r  c                 s   r  r]   ry  r
  r"   r"   r#   r    r  c                 s   r  r]   r  r
  r"   r"   r#   r    r  )Zretain_graph)+r   r  unittestSkipTestr   r@   r  r.   rY   floatr  r  r  r
   r~   r:   cudar  r,  r-  Zcopy_r  r  r  r   r  r   r-  rn  r  rp   r  rV   r  
randn_liker  r  Zautogradr  r  ry  r   r  )r^   r  	cpu_inputr  Zcpu_input_tupleZis_any_input_complexZgpu_input_tuple
cpu_module
gpu_moduleZ	cpu_paramZ	gpu_paramZcpu_pZgpu_p
cpu_output
gpu_outputr6   Zcpu_gradOutputZgpu_gradOutputcpu_gradInputgpu_gradInputZcpu_d_pZgpu_d_pZcpu_gradInputsZgpu_gradInputsZcpu_d_iZgpu_d_iZoutputs_cpuZoutputs_gpuZcpu_ggZgpu_ggr"   r"   r#   rF     s   








 $zModuleTest.test_cuda)rd   re   rf   r   r   rh   ri   r  r  r  r  r  rF   __classcell__r"   r"   r  r#   r  M  s    %r  c                   @   s   e Zd Zdd ZdS )InputVariableMixinc                    s    t | d} fdd  |S )NFc                    s@   t | tjr|  s|  rd| _| S t|  fdd| D S )NTc                 3       | ]} |V  qd S r]   r"   r  map_variablesr"   r#   r    r  zGInputVariableMixin._get_input.<locals>.map_variables.<locals>.<genexpr>)r  r.   r  is_floating_pointr  r  r  rq   r  r"   r#   r    s
   z4InputVariableMixin._get_input.<locals>.map_variables)r  r  )r^   r/  r"   r  r#   r    s   zInputVariableMixin._get_inputN)rd   re   rf   r  r"   r"   r"   r#   r    s    r  c                       s@   e Zd Z fddZdd Zdd Zdd Zed	d
 Z  Z	S )NewModuleTestc                    s   t  j|i | |dd| _|dd| _|dd| _|dd| _|dd| _|dd| _|d	d
| _	|dd| _
|dd| _|dd| _|dd | _|dd| _|dd| _d S )Nr  Fcheck_inplacerV   Tskip_doubler  r>   r?   rS   test_cpur  rW   r  r   supports_fwgrad_bwgrad)r  r  r1  r  r  rV   r   r  r>   r?   r!  r  rW   r  r   r"  r  r  r"   r#   r  $  s   zNewModuleTest.__init__c              	      s   t dd   D }t| fdd}| jr1dks J t|d } |d | nt||| | j	| j
| jd | jrXt||| | j	| j
| jd d S d S )	Nc                 s       | ]}|V  qd S r]   r"   r
  r"   r"   r#   r  5  rB  z1NewModuleTest._check_gradients.<locals>.<genexpr>c                     s   |rJ   | d  S r]   )r  )Zinputs_and_paramsrb   r  Z
num_inputsr  r"   r#   fn_to_gradcheck8  s   z7NewModuleTest._check_gradients.<locals>.fn_to_gradcheckr,   r   )rW   	fast_modeZcheck_forward_ad)rW   r&  Zcheck_fwd_over_rev)r  r  rQ  r  r.   r  r  
assertTruer   rW   r  r   rV   r   r"  )r^   r  r  input_tupleparamsr%  Ztest_input_jacobianr"   r$  r#   _check_gradients4  s&   zNewModuleTest._check_gradientsc                    s  t  }t d t|tr|n|f}|  |    | jrt|dks)J |d }| j	| j
ddi}|j}t   |}W d    n1 sKw   Y  |j| t|}	|	 }
t  ||
}W d    n1 sqw   Y  |
j| || |j  }|jd urt   |j  W d    n1 sw   Y  |	jd urt   |	j  W d    n1 sw   Y  || || |j|	j d fdd	}tdd |D rAtrAtd	d |D }     |  |t jjd t j dkr?td
d |D } d t jd  |  W d    n	1 s3w   Y  |t jjd n7dd fddfddfddtfdd|D }    |  |t j tfdd|D }    |  |t j trx| j rxtfdd|D }     |  |t jjd tdd |D } !   |  |t j tdd |D }    |  |t jjd | j"r t j#j"j$dd  |  |t jjd W d    n	1 sw   Y  t j dkr8tdd |D } d t jd  |  W d    n	1 s,w   Y  |t jjd | j%sXtfdd|D }     |  |t jjd | j&sxtfdd|D } '    |  |t jj(d t | d S )Nr,   r   ZinplaceTc                    s6      D ]}||  |d ur| | qd S r]   )r  ZassertIsInstancer  Z
get_device)Ztensor_typeZ	device_idr5   )r  r  r"   r#   assert_module_parameters_are{  s   z<NewModuleTest._do_test.<locals>.assert_module_parameters_arec                 s   s    | ]	}t |tjV  qd S r]   )r  r.   Z
LongTensorr  r"   r"   r#   r    r  z)NewModuleTest._do_test.<locals>.<genexpr>c                 s   r  r]   r  r  r"   r"   r#   r    r  c                 s       | ]}| d V  qdS r  r,  r  r"   r"   r#   r    r  c                 S   s(   |   r	| |S |  r| |S | S r]   )r  r   r  )r  realr  r"   r"   r#   to_type  s
   

z'NewModuleTest._do_test.<locals>.to_typec                    s    | t jd S r]   )r.   Zfloat16r]  r/  r"   r#   to_half  s   z'NewModuleTest._do_test.<locals>.to_halfc                        | t jt jS r]   )r.   float32Z	complex64r0  r1  r"   r#   	to_single  r  z)NewModuleTest._do_test.<locals>.to_singlec                    r3  r]   )r.   r)  r  r0  r1  r"   r#   	to_double  r  z)NewModuleTest._do_test.<locals>.to_doublec                 3   r  r]   r"   r  r5  r"   r#   r    r  c                 3   r  r]   r"   r  r6  r"   r#   r    r  c                 3       | ]	} |  V  qd S r]   r,  r  r7  r"   r#   r    r  c                 s   r  r]   )cpur  r"   r"   r#   r    r  c                 s   r  r]   r,  r  r"   r"   r#   r    r  F)enabledrE   c                 s   r-  r  r,  r  r"   r"   r#   r    r  c                 3   r9  r]   r,  r  r8  r"   r#   r    r  c                 3   r9  r]   r,  r  )r2  r"   r#   r    r  r]   ))r.   Zget_num_threadsZset_num_threadsr  r  r*  __repr__r  rQ  r~   r:   _versionr   r  r   rn  ZassertNotEqualr-  r  r  Zno_gradrV  backwardallr   r  r  ZFloatTensorZdevice_countrm  rY   ZDoubleTensorr  r:  r  backendsflagsr   r  halfZ
HalfTensor)r^   r  r  r/  Znum_threadsr(  Z	module_ipZinput_versionr<  Zinput_ipZinput_ip_cloneZ	output_ipr  r+  r"   )r  r  r6  r2  r5  r/  r#   r  P  s   















zNewModuleTest._do_testc                 C   r  Nr   Fr  r  r"   r"   r#   _get_target  r  zNewModuleTest._get_targetc                 C   r  Nr:   Fr  r  r"   r"   r#   r:     r  zNewModuleTest.constructor_args)
rd   re   rf   r  r*  r  rD  r  r:   r  r"   r"   r  r#   r  #  s     r  c                       s\   e Zd ZejdhZ fddZdd ZdddZd	d
 Z	e
dd Ze
dd Z  ZS )CriterionTestr   c                    s   t  j|i | |dd| _|dd| _|dd| _|dd| _|dd| _|dd| _|d	d| _	|d
d| _
|dd| _|dd| _|dd | _| jd u rbt | _d S d S )NrF   Tr  FrV   r  check_bfloat16check_complexr!  r>   r?   rS   rW   r@   )r  r  r1  r  r  rV   r  rG  rH  r!  r>   r?   rW   r@   r.   r  r  r  r"   r#   r    s   
zCriterionTest.__init__c           
         sR  t | j | j| j  |  }   t  |  }| jd urD|j	 ||| j
d}t|t|f| j
  f }| j| }||| | jrP	 W d    d S tdd   D }t|tso|f| |f } fdd}	n|| |f } fdd}	t|	|| jd | jrt|	|| jd W d    d S W d    d S 1 sw   Y  d S )Nr  c                 s   r#  r]   r"   r
  r"   r"   r#   r    rB  z)CriterionTest.__call__.<locals>.<genexpr>c                    s
    | |S r]   r"   )r/  r   r)  r  r"   r#   apply_fn  s   
z(CriterionTest.__call__.<locals>.apply_fnc                    s    | ||S r]   r"   )rc  rd  r   r)  rJ  r"   r#   rK    r  )rW   )r   r@   r~   r:   r  r<  r  rD  r=   _forward_criterionr  r   r  r  r  r  r  r   rW   rV   r   )
r^   r  r/  r   r  Zref_argsr  r)  ZinputsrK  r"   rJ  r#   r     s4   


!"zCriterionTest.__call__Nc                    s  d fdd	 t r| jstdt| j |  }|  }| j| j	 }| j| j	 } ||d}|
 s9| r> ||}|| || t|}t|}	|  |tjtjhv rj|  }|  }| j| j	 }|j||||d}
|j|||	|d}|j|
||tjtjhv rdndd	dd
 |j|||
||d}|j||||	|d}|j|||tjtjhv rdndd	dd
 W d    d S 1 sw   Y  d S )NFc                    sJ   t | tjr|  j dS t | tr#t fdd| D S | S )Nr   c                 3   s    | ]	} |V  qd S r]   r"   r  )convert_dtyper   r  r"   r#   r  *  r  zACriterionTest.test_cuda.<locals>.convert_dtype.<locals>.<genexpr>)r  r.   r  r  r   requires_grad_r  )r  r   r  rM  r  r#   rM  &  s
   
z.CriterionTest.test_cuda.<locals>.convert_dtyper  TrI  rH   g-C6:?r   r  r  )r   r  r  r  r   r@   r  rD  r~   r:   r  r  r  r
   r  r.   rB  bfloat16rL  r  Z_backward_criterion)r^   r  r   r  r  Z
cpu_targetr  r  Z	gpu_inputZ
gpu_targetr  r  r  r  r"   rO  r#   rF   %  sF   






"zCriterionTest.test_cudac                 C   r  rC  r  r  r"   r"   r#   rD  X  r  zCriterionTest._get_targetc                 C   r  rE  r  r  r"   r"   r#   r:   [  r  zCriterionTest.constructor_argsc                 C   r  )Nr  Fr  r  r"   r"   r#   r  _  r  zCriterionTest.extra_argsr]   )rd   re   rf   r  r  unionr  r  rF   rD  r  r:   r  r  r"   r"   r  r#   rF    s    
%3
rF  r"   r   c                 C   s   t j|t j|dd}|d urt j|t j|d|   }||}t j||d}|| | }	|	   }
| }|	|
}|| | j
||||dd | j
|jj|
jj||dd d S )NT)r   rm  r  rl  )rm  Fr  )r.   r   r4  rw   rP  r  rN  r  r>  r  r  r  r-  )r  oprm  Zinp_dimsprecr  rc  Zout1Zgrad_input1Zop_bfp16rd  Zgrad_input2Zout2r"   r"   r#   _test_bfloat16_opsd  s   

 rT  c                 C   s   |s| d ||}|st|}|| |r#| | |  |sF| D ]}|jr9| |jt	|j q)| |jt	| d S d S )NT)
rN  r.   Z	rand_liker>  r  rZ   r  r  r  r  )r  r  r  Z
check_sizeZ	inferencer  ZgOr5   r"   r"   r#   _test_module_empty_inputw  s   


rU  c                     sJ   G dd dt j G  fdddt j}   }|  }t ||}|||fS )Nc                       s   e Zd Zd fddZ  ZS )z _create_basic_net.<locals>.Layerr  Nc                    s:   t    ttdd| _ttdddd| _	d S )NrN   rJ   r,   r  )
r  r  rh   r  r.   r  Zlayer_dummy_paramBufferrZ  Zlayer_dummy_bufr  r  r"   r#   r    s   
z)_create_basic_net.<locals>.Layer.__init__r  Nrd   re   rf   r  r  r"   r"   r  r#   Layer  s    rY  c                       s    e Zd Zd fddZ  ZS )z_create_basic_net.<locals>.Netr  Nc                    sB   t      | _ttdd| _tt	dddd| _
d S )NrN   rJ   r  r,   )r  r  l1rh   r  r.   r  Zdummy_paramrV  rZ  Z	dummy_bufr  )rY  r  r"   r#   r    s   
z'_create_basic_net.<locals>.Net.__init__rW  rX  r"   rY  r  r#   Net  s    r\  )rh   ri   Z
Sequential)r\  lrW  r  r"   r[  r#   _create_basic_net  s   
r^  )r  F)Nr6  r  )Nr  r   )Nr6  r  r   )r  r  )r  )r  r  )r,   r,   Nr  )r   r  )r  rE   rf  Fr  )r"   r   Nr  )abcr   r  r  copyr   	functoolsr   r   	itertoolsr   operatorr   r.   Z
torch.cudaZtorch.nnrh   Ztorch.nn.functionalZ
functionalro   r   r   Z$torch.testing._internal.common_utilsr	   r
   r   r   r   r   r   r   Z#torch.testing._internal.common_cudar   r   Ztorch.autograd.gradcheckr   r   Ztorch.autogradr   Ztorch.typesr   Ztorch.backends.cudnntypingr   r   r   collections.abcr   r  r  r$   r'   r   rY   Zmodule_testsr\   rj   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r(  r.  r0  r6  r8  r9  r;  r<  r=  rB  rD  rJ  rK  rO  r[  r]  rd  re  rl  rq  rs  r{  r|  r  r  r  r  r  r  r  r4  r5  r?  rE  rH  rI  rM  rO  rP  rS  rX  rZ  r[  r\  r^  re  rj  rk  rs  r   __annotations__Zcriterion_testsrz  Zregression_criterion_no_batchZ
reductionsr{  r   Zregression_test_infor0  Z!classification_criterion_no_batchr  r  Zclassification_cpp_parityr   r  r1  Zclassification_test_infor3  r2  r  r  r  r  r  rF  rT  rU  r^  r"   r"   r"   r#   <module>   s  (>
	<            s(

&
"		


	
{M H Iy