o
    wZh|\                    @   s&  U d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
mZ d dl	Z	d dlZd dlm  mZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d dl@mAZA d dlBZBe jjCe jDjjEjCe jDjjFjCe jDjjGjCe jDjjGjCgZHeIeA eJd< e jjKe jjLe jjMe jDjjGjNe jDjjGjNhZOePeQ eJd< g e
Rdd eHD ZSeIeQ eJd< dd eSD ZSi ZTeUeQeVf eJd< eHD ](ZWeWjXD ]!ZYeZeWeYZ[eWj\]dd]ddZ^e[eTvr2e^ deY eTe[< qqeddZ_G dd deZCdd Z`G d d! d!ZaG d"d# d#ZbG d$d% d%eZcG d&d' d'ZdG d(d) d)Zed*d+ Zfd,d- Zgd.d/ Zhd0d1 Zid2d3 Zjd4d5 Zkd6d7 Zld8d9 Zmd:d; Znd<d= Zod>d? Zpd@dA ZqdBdC ZrdDdE ZsdFdG ZtdHdI ZudJdK ZvdLdM ZwdNdO ZxdPdQ ZydRdS ZzdTdU Z{dVdW Z|dXdY Z}dZd[ Z~d\d] Zd^d_ Zd`da Zdbdc Zddde Zdfdg Zdhdi Zdjdk Zdldm Zdndo Zdpdq Zdrds Zdtdu Zdvdw Zdxdy Zdzd{ Zd|d} Zd~d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZddÄ Zddń ZddǄ ZddɄ Zdd˄ Zdd̈́ Zddτ Zddф Zddӄ ZddՄ Zddׄ Zddل ZdUdd܄Zddބ Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Ze(ejddeoe= dd e(ejddeo*e= dd e(ejddeo9e= dd e(ejddeoGe=e jfdde(e&dgdddde(e&dgdd	ddfZѐd
d ZҐdd ZӐdd ZԐdd ZՐdd Z֐dd Ze jj٠ڡ oe jj٠ېdd Zg eee jjeve(e%fdeee jje<ewe(e%e(ejddeBߐddfdeee jje<exe(eddde(e%fdeee jjeydeee jje<ezdeee jje<e{e(eddde(e%fdeee jjesdeee jjete(ejddeBߐddd e(e;de jgdd fdeee jjeue<e(eddde(e%fd!eee jjd"e|e(ejϐd#d$eBߐdde(ejϐd#d%eBߐddfd&eee jjd"e}e(eddeBߐdde(ejϐd#d$eBߐdde(ejϐd#d%eBߐddfd&eee jjd"e~e(e%e(ejϐd#d$eBߐdde(ejϐd#d%eBߐddfd&eee jjee(ejdd'de jgd(fdeee jjeed)dڐd*e<d"e(e!d+d,dde(e"dde jgd-e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jjeed0dڐd*e<d"e(e!d+d,dde(e"dde jgd-e(ejddde jgd(e(ejddde je jgd(e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jjeed1dڐd*e<d"e(e!d2d,dde(e"dde jgd-e(e%dde(ejddfe(e#e jd.iddfd/eee jjeed)dڐd"d3e<d"ee je(e!d+d,dde(e"dde jgd-e(ejdd4e jfdd e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.idde(e#e jd5iddfd6eee jjeed0dڐd"d3e<d"ee je(e!d+d,dde(e"dde jgd-e(ejdde je je jfd-e(ejddde je jgd(e(ejddde je jgd(e(ejdd4e jfdd e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.idde(e#e jd5iddfd6eee jjeed1dڐd"d3ee je<d"e(e!d2d,dde(e"dde jgd-e(e%e(ejdde(ejddde je jge=d7e(ejdd4e jfdd fe(e#e jd.idde(e#e jd.idd4e(e#e jd5iddfd8eee jjee(edddfdeee jjee(ejdd'de jgd(fdeee jjee<e(e%e(edddfd!eee jjee<e(e%e(edddfd!eee jjee(edddfdeee jjee(eddde(e:dde jgdd fdeee jjeed)d"d*e<d"e(e!d+d,dde(e"dde jgd-e(e$e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jj eed0d"d*e<d"e(e!d+d,dde(e"dde jgd-e(e$e(ejddde jgd(e(ejddde je jgd(e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jjeed1d"d*e<d"e(e!d2d,dde(e"dde jgd-e(e$e(e%e(ejddfe(e#e jd.iddfd/eee jjeed)d"d"d3e<d"e(e!d+d,dde(e"dde jgd-e(e$e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jjeed0d"d"d3e<d"e(e!d+d,dde(e"dde jgd-e(e$e(ejddde jgd(e(ejddde je jgd(e(e;ddde jgd(e(e;ddde jgd(fe(e#e jd.iddfd/eee jjeed1d"d"d3e<d"e(e!d2d,dde(e"dde jgd-e(e$e(e%e(ejddfe(e#e jd.iddfd/eee jjefe(edddfdeee jjege(ee je d.d.d9e je d.d.d9idd:d;dge(eddde(ejdd:de jgd(fd<eee jjee(eddde(edddfdeee jjee(eddde(eddde(ejddeBߐddd fdeee jj	ee(eddde(eddde(eddde(e:ddfdeee jj
edeee jjNedeee jjee<e(e:ddfd!eee jjehe(eddde(eddd4e(eddde(edddfdeee jjele(eddde(e;ddde jgd(e(ejdd:de jgd(fdeee jjee(edddfdeee jjee(eddde(e:ddde(edddfdeee jjee(eddde(e:ddde(edddfdeee jjee(eddde(ejdd:de jgd(fdeee jjee(edddfdeee jjeie(eddde(ejdd:de jgd(fdeee jjeje(edddfdeee jjeke(edddfdeee jjee(edddfdeee jjee(eddde(ejdd:de jgd(fdeee jjee(eddde(e:de jgdd fdeee jjee(eddde(e:de jgdd fdeee jjeed"dڐd=e(ejdde(ee je d>d?d9idd:e jgd;d e(ejdd4e jgdd fd@eee jjee(eddde(e:ddde(eddde(eddde(edddfdeee jjee(ejdd:de jgd(fdeee jjedeee jjeed"d"dAe(eddd4e(ee je d.d.d9iddd;de(eddddde(eddddde(eddde=dd fdBeee jjee(ejdd:dde(ejddCdde(ejdddde(ejdddde(ejddDddfdeee jj eeܐr;dne(ejddeBߐddd fdڐdEeee jj!edeee jj"eed)dFd"e(edddfdGeee jj#eed0dFd"e(edddfdGeee jj$eed1dFd"e(edHde(edHde(edHd:e(edHdIe(edHdDe(edddfdGeee jj%ee(edde(edde(edd:e(eddCe(eddIe(eddDfdeee jj&ee(edddfdeee jj'edeee jj(d"ee(edddd;dge(eddde(ejddJfdKeee jj)d"ee(ee je d.d.d9iddd;e>dLe(ee je d.dMd9idd:dde(edddd;dge(edddfdKeee jj*ee(edddd;dge(edddfd<eee jj+ed"e(edddd;dge(edddfdNeee jj,d"ee(edddfd&eee jj-ee(ee je d.d.d9iddddge(edddfd<eee jj.eeܐr;dne(ejddeBߐddd fdeee jj/edeee jj0ee(e%fdeee jj1ee(e%fdeee jj2eed"dOeҐdPeee jj3eeҐdPeee jj4eeӐdPeee jj5eeܐrdne(ejddeBߐddd fdeee jj6ee(ejdd:de jgd(fdeee jj7edeee jj8edeee jj9ee(eddde(ejdd:de jgd(fdeee jj:ee(eddde(ejdd:de jgd(fdeee jj;ee(edddfdeee jj<ee(e%fdeee jj=ee(e%fdeee jj>edeee jj?eeܐrvdne(ejddeBߐddd fdeee jj@eeܐrdne(ejddeBߐddd fdeee jjAee(e%fdeee jjBee(e%fdeee jjCd"eed"dOeeѐdQeee jjDd"eedڐdOeeѐdQeee jjEd"eee(e%feѐdReee jjFedeee jjGee<e(eddddde(edddddfd!eee jjHee<e(eddddde(edddddfd!eee jjIeÐdeee jjJee<e(eddddde(edddddfd!eee jjKee<e(eddddde(edddddfd!eee jjLee(e%fdeee jjMeƐdeee jjNee(ejddddfdeee jjOee(ejddddfdeee jjPeeՐdPeee jjQee֐dPeee jjReee(ejddfdSeee jjSeɐdeee jjTee(ejddddfdeee jjUee(ejddddfdZVeIee eJdT< dS (V      N)deepcopy)Enum)wrapspartial)chainproduct)pack_padded_sequence)make_tensor)
TEST_CUDNN)floating_typesfloating_and_complex_types_andget_all_fp_dtypes)_TestParametrizer_update_param_kwargsexpectedFailureMPStoleranceOverridetolskipCUDAIfCudnnVersionLessThanskipCUDAIfRocmprecisionOverrideskipMetaskipMPSskipCUDAVersionIn)DecorateInfo)cosineembeddingloss_referencecross_entropy_loss_referencectcloss_referencehingeembeddingloss_referencehuberloss_referencekldivloss_referencemarginrankingloss_referencemultimarginloss_referencemultilabelmarginloss_referencenllloss_referencenlllossNd_referencesmoothl1loss_referencesoftmarginloss_referenceget_reduction)freeze_rng_state	skipIfMPSskipIfMPSOnMacOS13GRADCHECK_NONDET_TOLTEST_WITH_ROCM
IS_WINDOWSskipIfTorchDynamo)
ModuleTypeMODULE_NAMESPACESMODULES_TO_SKIPc                    s    g | ]  fd d j D qS )c                    s   g | ]}t  |qS  )getattr).0module_name	namespacer2   U/var/www/auris/lib/python3.10/site-packages/torch/testing/_internal/common_modules.py
<listcomp>5       z<listcomp>.<listcomp>)__all__)r4   r2   r6   r8   r9   4   s    r9   MODULE_CLASSESc                 C   s   g | ]}|t vr|qS r2   )r1   )r4   clsr2   r2   r8   r9   7       MODULE_CLASS_NAMESztorch. z.modules.TrainEvalMode)
train_only	eval_onlytrain_and_evalc                   @   s2   e Zd ZdZdejdfddZdd Zdd	 ZdS )
moduleszQ PROTOTYPE: Decorator for specifying a list of modules over which to run a test. NTc                 C   s0   t || _|d urt|nd | _|| _|| _d S N)listmodule_info_listsetallowed_dtypestrain_eval_modeskip_if_dynamo)selfZmodule_info_iterablerK   rL   rM   r2   r2   r8   __init__M   s   

zmodules.__init__c                 C   s^   g }| j tjks| j tjkr|d | j tjks| j tjkr$|d |js-|d d }|S )NTF   )rL   rB   rC   rE   appendrD   train_and_eval_differ)rN   module_infotraining_flagsr2   r2   r8   _get_training_flagsT   s   

zmodules._get_training_flagsc                 #   s>   |d u r	t d| jD ]}t||j}| jd ur!|| j}| |}t||D ]p\}}|j	}	t
|dkrC|	d|r>dnd 7 }	d|i}
t|
d| t|
d| z.t  fd	d
}| jrmtjjjjsmtd|}t|j|j j|j|}||	|
|fV  W q+ ty } ztd|	 d|j d |d }~ww qd S )NzThe @modules decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests()rP   _Z
train_modeZ	eval_moderS   dtypetrainingc                     s    | i |S rG   r2   argskwargstestr2   r8   test_wrapper~      z/modules._parametrize_test.<locals>.test_wrapperz/Policy: we don't run ModuleInfo tests w/ DynamozFailed to instantiate z for module !)RuntimeErrorrI   rJ   supported_dtypesdevice_typerK   intersectionrU   r   formatted_namelenr   r   rM   torchtesting	_internalZcommon_utilsZTEST_WITH_TORCHINDUCTORr.   r   get_decorators__name__	Exceptionprintname)rN   r]   Zgeneric_clsZ
device_clsrS   dtypesrT   rX   rW   	test_nameparam_kwargsr^   Zdecorator_fnexr2   r\   r8   _parametrize_testd   s>   




zmodules._parametrize_test)	rk   
__module____qualname____doc__rB   rE   rO   rU   rs   r2   r2   r2   r8   rF   J   s    
rF   c                 C   s   | t v rt |  S | jS rG   )r?   rk   )
module_clsr2   r2   r8   get_module_common_name   s   rx   c                   @   s    e Zd ZdZddgZdd ZdS )FunctionInputz: Contains args and kwargs to pass as input to a function. rZ   r[   c                 O   s   || _ || _d S rG   rY   )rN   rZ   r[   r2   r2   r8   rO      s   
zFunctionInput.__init__Nrk   rt   ru   rv   	__slots__rO   r2   r2   r2   r8   ry      s    ry   c                   @   s"   e Zd ZdZg dZdddZdS )ModuleInputzA Contains args / kwargs for module instantiation + forward pass. constructor_inputforward_inputdescreference_fnNr@   c                    sB   || _ || _|| _ | _ d urt  fdd}|| _d S d S )Nc                    s2   t |t |}} | t|  g|R i |S rG   )r   rH   
parameters)mrZ   r[   r   r2   r8   copy_reference_fn   s    z/ModuleInput.__init__.<locals>.copy_reference_fn)r~   r   r   r   r   )rN   r~   r   r   r   r   r2   r   r8   rO      s   
zModuleInput.__init__)Nr@   Nrz   r2   r2   r2   r8   r|      s    r|   c                   @   s   e Zd ZdZdZdZdS )ModuleErrorEnumz7 Enumerates when error is raised when testing modules. r   rP   N)rk   rt   ru   rv   CONSTRUCTION_ERRORFORWARD_ERRORr2   r2   r2   r8   r      s    r   c                   @   s*   e Zd ZdZg dZejedddZdS )ErrorModuleInputzw
    A ModuleInput that will cause the operation to throw an error plus information
    about the resulting error.
    module_error_inputerror_on
error_typeerror_regex)r   r   c                C   s   || _ || _|| _|| _d S rG   r   )rN   r   r   r   r   r2   r2   r8   rO      s   
zErrorModuleInput.__init__N)	rk   rt   ru   rv   r{   r   r   ra   rO   r2   r2   r2   r8   r      s    r   c                   @   sl   e Zd ZdZdde ejejfejejfddddddddd	Z	d
d Z
dd Zedd Zedd ZdS )
ModuleInfoz+ Module information to be used in testing. r2   NT        F)skips
decoratorsro   dtypesIfMPSdtypesIfHpusupports_gradgradgradcheck_nondet_tolmodule_memformat_affects_outrR   module_error_inputs_funcgradcheck_fast_modec                C   sz   || _ || _g |r|ng |r|ng R | _|| _|| _|| _|| _|	| _|
| _|| _	|| _
|| _t|tjjjj| _d S rG   )rw   module_inputs_funcr   ro   r   r   r   r   r   rR   r   r   
issubclassrg   nnrF   lazyZLazyModuleMixinZis_lazy)rN   rw   r   r   r   ro   r   r   r   r   r   rR   r   r   r2   r2   r8   rO      s    zModuleInfo.__init__c                 C   sH   g }| j D ]}t|tr||||||r||j  q|| q|S rG   )r   
isinstancer   	is_activeextendrQ   )rN   Z
test_classrp   devicerW   rq   result	decoratorr2   r2   r8   rj      s   

zModuleInfo.get_decoratorsc                 C   s"   |dkr| j S |dkr| jS | jS )NmpsZhpu)r   r   ro   )rN   rc   r2   r2   r8   rb      s
   zModuleInfo.supported_dtypesc                 C   s
   t | jS rG   )rx   rw   rN   r2   r2   r8   rn     s   
zModuleInfo.namec                 C   s   | j ddS )NrA   rV   )rn   replacer   r2   r2   r8   re   	  r_   zModuleInfo.formatted_name)rk   rt   ru   rv   r   rg   float16float32Zbfloat16rO   rj   rb   propertyrn   re   r2   r2   r2   r8   r      s(    

!

r   c                 K   s|   t t|||d}ttddt|dddd dttddd	d
t|dddd dttddt|dddd dg}|S )Nr   rW   requires_grad
         r   )inputc                 S   s,   t ||d  |d dddd S )Nr   rP   r   r   )rg   mmtviewexpand)r   pr   r2   r2   r8   <lambda>  s   , z/module_inputs_torch_nn_Linear.<locals>.<lambda>r~   r   r   Fbiasno_biasc                 S   s   t ||d  S )Nr   )rg   r   r   r   r   ir2   r2   r8   r         r}         no_batch_dimc                 S   s*   t |dd|d  d|d  S )NrP   r   r   )rg   r   r   r   r   r2   r2   r8   r        * r   r	   r|   ry   rS   r   rW   r   rX   r[   
make_inputmodule_inputsr2   r2   r8   module_inputs_torch_nn_Linear  s"   



r   c              	      s   t t|||d}ddd ttdddt|d|d	 d
ttdddddt|d|d	d fdddttdddt|d|dd fdddg}|S )Nr   Tc                 S   sl   t d||d |}|r4|jd dkr|d|d  }|S ||d dd|jd |d jd  }|S )Nzbn,anm,bm->bar   rP   r   )rg   Zeinsumshaper   r   )r   r   x1x2r   r   r2   r2   r8   bilinear_reference_fn&  s   ,z>module_inputs_torch_nn_Bilinear.<locals>.bilinear_reference_fn   r   r   )r   r   )r   r   r   Fr   r   c                    s    | |||ddS )NFr   r2   r   r   r   r   r   r2   r8   r   6  s    z1module_inputs_torch_nn_Bilinear.<locals>.<lambda>r}   r   c                    s    | || dd| ddS )NrP   r   )r   r   r   r2   r8   r   :      )Tr   r   r2   r   r8   module_inputs_torch_nn_Bilinear#  s$   



r   c              	   K   s  t t|||d}di fdddifdddifddd	ifd
d
difg}g }|D ]^\}	}
|
fdd}|d }|d
dr?|dn|d }|ttdi |
t|||	|d |d }|d
drh|dn|d }|ttdi |
t||d|	 |d q%|S )Nr   r@   reduction_sum	reductionsumZreduction_batchmeanZ	batchmeanreduction_nonenoneZ
log_targetTc                 S      t ||fi |S rG   )r   r   r   r   r   constructor_kwargsr2   r2   r8   r   M     z6module_inputs_torch_nn_KLDivLoss.<locals>.reference_fnr   r   Fr}   r2   scalar_)r   r	   loggetrQ   r|   ry   )rS   r   rW   r   rX   r[   r   casesr   r   r   r   r   targetZscalar_inputZscalar_targetr2   r2   r8    module_inputs_torch_nn_KLDivLoss@  s<   



 r   c                 K   s  |||fdd}t t||dd}di fdddifd	dd
ifdddifdd|d ifd|d ddfd|d ddfg}g }	|D ]\}
}|fdd}|	ttd#i |t|dtjd|d 	d
  |
|d |fdd}|	ttd#i |t|dtjddd|d 	d
  d|
 |d |	ttd#i |t|dtjddddd|d 	d
  d |
 |d |	ttd#i |t|d!tjdd|d 	d
  d"|
 |d qE|	S )$Nc                 S   s   t | ||ddjdd|S )NFr   rP   dim)r	   log_softmaxrequires_grad_)r   r   rW   r   r2   r2   r8   r   i  s   z2module_inputs_torch_nn_NLLLoss.<locals>.make_inputFr   r@   r   r   r   r   r   ignore_indexr   weightsweightr   Zweights_ignore_index)r   r   Zweights_ignore_index_negr   c                 S   r   rG   )r#   r   r2   r2   r8   r     r   z4module_inputs_torch_nn_NLLLoss.<locals>.reference_fn)   r   r   r   r}   c                 S   r   rG   )r$   r   r2   r2   r8   nd_reference_fn  r   z7module_inputs_torch_nn_NLLLoss.<locals>.nd_reference_fn)r   r   r   r   r   Znd_)r   r   r   r   r   r   higher_dim_)r   r   r   3d_r2   )r   r	   absrQ   r|   ry   rg   emptyuniform_mulfloorlong)rS   r   rW   r   rX   r[   r   make_weightr   r   r   r   r   r   r2   r2   r8   module_inputs_torch_nn_NLLLossh  sl   


"	&	 	r   c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}g }	|D ] \}
}|	ttdi |t|d|d|d |
td q(|	S )Nr   Fr@   r   r   r   reduction_meanmeanr   r   r   rP   r}   r2   )r   r	   rQ   r|   ry   r   no_batch_dim_reference_fn)rS   r   rW   r   rX   r[   r   make_targetr   r   r   r   r2   r2   r8   &module_inputs_torch_nn_GaussianNLLLoss  s(   



	r   c              
      s   t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifdddifddddfg}ddd g }	|D ]:\}
}|f fdd	}|dd}|rU|dn|d d}|	ttdi |t||d 	 |
|d q=|	S )Nr   Fr@   r   r   r   r   r   r   r   fullTZno_log_input	log_inputZfull_no_log_input)r   r   :0yE>c                 S   s   |r|   ||  }n| || |   }|r5||| | ddtj |    |dkd7 }|dkr;|S |dkrG| |   S | S )N      ?       @rP   r   r   r   )expr   r   mathpiZmasked_fillr   numel)r   r   r   r   r   epsr   r2   r2   r8   poissonnllloss_reference_fn  s   8zJmodule_inputs_torch_nn_PoissonNLLLoss.<locals>.poissonnllloss_reference_fnc                    s    ||fi |S rG   r2   r   r  r2   r8   r     r   z;module_inputs_torch_nn_PoissonNLLLoss.<locals>.reference_fnr   r   r   r   MbP?r}   )TFr   r   r2   )
r   r	   r   r   addrQ   r|   ry   Zfloor_Zabs_)rS   r   rW   r   rX   r[   r   r   r   r   r   r   r   r   r   r2   r  r8   %module_inputs_torch_nn_PoissonNLLLoss  s4   






r	  c                 K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}ddd}	g }
|D ]A\}}|
ttdi |t|d|d|t |	fi |d |
ttdi |t|d|d| dt |	fi |d q-|
S )Nr   Fr@   r   r   r   r   r   r   r   c                 S   sJ   |dkr||  dS |dkr||  d |  S ||  d S )Nr   r   r   )powr   r  )r   r   r   r   r   r2   r2   r8   mse_loss_reference_fn  s
   z=module_inputs_torch_nn_MSELoss.<locals>.mse_loss_reference_fnr  r}   r2   Z_scalar)r   r   r	   rQ   r|   ry   )rS   r   rW   r   rX   r[   r   r   r   r  r   r   r   r2   r2   r8   module_inputs_torch_nn_MSELoss  s:   



r  c                    s   fdd}|ddrdnd |dd}|d	d
}|dur@t |ts$J  D ]\}}||v r?|dur?|| }	||	|< q( fdd|D }
t  | |
i  }W d   n1 saw   Y  |rut| }|dkru|dS |S )a  Reference function for modules supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.

    Currently it only supports modules which return a single Tensor as output.
    You can bind the following kwargs.
    Kwargs:
        batch_first[bool] : If True, all the Tensors in `args` while be unsqueezed at dim `0` .
                        and output will be squeezed at dim `0` else dim `1` for both.
        kwargs_to_batchify[dict] : Dictionary specifying the name of the argument and dimension to unsqueeze.
                               Useful if there are few arguments whose batch dimension are different
                               from the ones selected by `batch_first`.
        is_criterion[bool] : Specify if the module is a criterion and handle the reduction for output accordingly.
    c                    s"     | |}|  v r |  |S rG   )r   pop)keydefaultv)r[   r2   r8   get_and_pop-  s   
z.no_batch_dim_reference_fn.<locals>.get_and_popbatch_firstTr   rP   kwargs_to_batchifyNis_criterionFc                       g | ]}|  qS r2   	unsqueezer4   r   	batch_dimr2   r8   r9   >  r:   z-no_batch_dim_reference_fn.<locals>.<listcomp>r   )r   dictitemsr  r(   squeezer'   )r   r   rZ   r[   r  r  r  kr  Zbdimsingle_batch_input_argsoutputr   r2   )r  r[   r8   r     s(   


r   c                    s   | ddrdnd d|v r|d d|v r&|d dur&|d d|d<  fdd|D }t  | |i |}|d  |d dfW  d   S 1 sRw   Y  dS )	a  Reference function for MultiheadAttention supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    r  Tr   rP   key_padding_maskNc                    r  r2   r  r  r  r2   r8   r9   V  r:   z.no_batch_dim_reference_mha.<locals>.<listcomp>)r   r  r  r(   r  )r   r   rZ   r[   r   r!  r2   r  r8   no_batch_dim_reference_mhaI  s   
$r#  c           	      O   s   t |dkr|\}d}nt |dkr|\}}|d}|d r!dnd}|d ||}||f}t  | |i |}|d ||d dfW  d   S 1 sTw   Y  dS )a  Reference function for RNN and GRU supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rP   Nr   r  r   rf   r  r  r(   r  	r   r   rZ   r[   inphr  r   r!  r2   r2   r8   no_batch_dim_reference_rnn_gru\  s   


$r(  c           	      O   s   t |dkr|\}d}nt |dkr$|\}}|d d|d df}|d r*dnd}|d ||}||f}t ( | |i |}|d ||d d d|d d dffW  d   S 1 shw   Y  dS )a  Reference function for LSTM supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rP   Nr   r   r  r$  r%  r2   r2   r8   no_batch_dim_reference_lstmt  s   

0$r)  c           	      O   s~   |\}\}}| d| d| dff}t  | |i |}|d d|d dfW  d   S 1 s8w   Y  dS )zReference function for LSTMCell supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r   rP   N)r  r(   r  )	r   r   rZ   r[   r&  r'  cr   r!  r2   r2   r8   no_batch_dim_reference_lstmcell  s   $r+  c                    s    fdddD S )Nc              	      s>   g | ]}t t|d t d dttddd| dqS ))r   r   r   Tr  no_batch_dim_r~   r   r   r   )r|   ry   r   r   )r4   r   r   r2   r8   r9     s    
z8generate_regression_criterion_inputs.<locals>.<listcomp>)r   r   r   r2   r0  r2   r0  r8   $generate_regression_criterion_inputs  s   
r1  c                 K   s|   t t|||d}ttddt|ddtdttdt|ddttd	d	t|dd
dttdddt|dddgS )Nr   r   )kernel_size)r      r   r}   )r   r   r3  r~   r   r   strider~   r   r   rP   
stride_padr   r	   r|   ry   r   rS   r   rW   r   rX   r[   r   r2   r2   r8    module_inputs_torch_nn_AvgPool1d  s$   





r;  c                 K   s   t t|||d}ttdt|ddtdttdt|ddttddt|ddd	ttddd
t|ddd	ttdddt|ddd	ttddddt|ddd	ttddd
ddt|ddd	gS )Nr   r   r   r   r3  r3  r   r}   r   r   r3  r3  r4  r6  r7  rP   rP   r8  rP   Zdivisor_overridedivisordivisor_stridedivisor_stride_padr9  r:  r2   r2   r8    module_inputs_torch_nn_AvgPool2d  s<   







rD  c                 K   s  t t|||d}ttdt|ddtdttdt|ddttddt|d	d
dttdddt|d	ddttdddt|d	ddttdddt|dddttdddt|dddttdddt|dddttdddt|dddttddddt|d	ddttdddddt|d	ddttdddddt|d	ddttdddddt|dddttdddddt|dd dttdddddt|dd!dgS )"Nr   r   r   r   r   r   r   r   r   r}   r   r   r   r   r   r4  r   r   r   r   r   r   r6  r7  rP   rP   rP   r8  r   )rP   r   rP   Zstride_pad_gpu_fixedkw_output)r   r   r   rP   )rP   rP   r   )r   r   r   r   r   Zstride_pad_gpu_general_outputr   r   Zstride1_pad0_gpu_inputZstride_pad_gpu_input_nooverlapr@  rA  rB  rC  Z%divisor_stride_pad_gpu_fixedkw_outputZ%divisor_stride_pad_gpu_general_outputZdivisor_stride1_pad0_gpu_inputZ&divisor_stride_pad_gpu_input_nooverlapr9  r:  r2   r2   r8    module_inputs_torch_nn_AvgPool3d  s|   















rJ  c                 K   s^   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dgS )Nr   r   rP   r   r   singler7  r   r   r   r/  rP   Z
one_outputr9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveAvgPool1d  s   


rN  c              	   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddttdt|dddgS )Nr   r   rP   r   r   r3  rL  r7  r   r   r3  r   r/  rP   Zsingle_1x1outputr   r   tupler   N
tuple_noner9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveAvgPool2d  .   




rU  c              	   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|d
ddttdt|d
ddttdt|dddgS )Nr   r   )r   r   r   r      rL  r7  )r   r   r   rW  r   r/  r   r   r   )r   r   r   r   rW  rR  )Nr   r   rT  )r   r   r   )rP   rP   r   r   r3  Zlast_dimr9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveAvgPool3d*  rV  rY  c                 K   sF   t t|||d}ttdt|dddttdt|dtddgS )	Nr   r   rK  rL  r7  rM  r   r/  r9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveMaxPool1d@  s   

rZ  c                 K   sv   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddgS )Nr   r   rO  rL  r7  rP  r   r/  rQ  rR  rS  rT  r9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveMaxPool2dM  s&   



r[  c              
   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddttdt|dddttd	t|dddgS )Nr   r   )r   r   r   r3  rW  rL  r7  )r   r   r3  rW  r   r/  rX  rR  )r   Nr   rT  )r   r      	   r   Zsingle_nonatomic)r   r   r3  r   r   Ztuple_nonatomicr9  r:  r2   r2   r8   (module_inputs_torch_nn_AdaptiveMaxPool3d`  s6   





r^  c                 K   s   t t|||d}ttdt|dddttdt|dddttdd	d t|dd
dttdd	ddt|dddttdd	dddt|dddttdd	ddt|dddttdd	ddt|dddgS )Nr   r   r   affiner7  r   )r   r   r   3d_inputr  Zaffine_simple_average333333?F
not_affineTnot_tracking_statsZ3d_input_not_affine)r   r   r]  
zero_batchr   r:  r2   r2   r8   "module_inputs_torch_nn_BatchNorm1dy  s<   






re  c                 K      t t|||d}ttdt|ddttddd t|dddttdddt|dd	dttdddd
t|dddttddddd
t|dddttdddd
t|dddgS )Nr   r   r>  r4  r  Z2d_simple_averager7  g?momentumFrb  Trc  r   ra  )r   r   r   r   rd  r   r:  r2   r2   r8   "module_inputs_torch_nn_BatchNorm2d  2   





rh  c                 K   rf  )Nr   r   rG  r4  r  Z3d_simple_averager7  ffffff?rg  Frb  Trc  r   ra  )r   r   r   r   r   rd  r   r:  r2   r2   r8   "module_inputs_torch_nn_BatchNorm3d  ri  rk  c           	         s   |d }| dd| dd}tt|||d|ri gni ddig}d\  ftd	d
 t|D  d  fddtddg|D S )NNr   F
transposedr   paddingZsamerX  c                 s   s    | ]}|d  V  qdS rS  r2   r4   r   r2   r2   r8   	<genexpr>  s    z0module_inputs_torch_nn_ConvNd.<locals>.<genexpr>r5  c                    sf   g | ]/\}}t rtfi |n	t fi |t|r!n|r(d nd|r-dntdqS )r@   r   Nr}   )r|   ry   r   )r4   Z
with_batchZconv_kwargsZC_inZC_outinput_batch_shapeinput_no_batch_shaper2  r   r   r2   r8   r9     s    


z1module_inputs_torch_nn_ConvNd.<locals>.<listcomp>T)r   r   r	   rR  range	itertoolsr   )	rS   r   rW   r   rX   r[   rl  rm  Zconv_kwargs_listr2   rq  r8   module_inputs_torch_nn_ConvNd  s   
rv  c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifg}g }	|D ]&\}
}|fdd}|	ttdi |t|d|d|d |
|d q-|	S )Nr   Fr@   r   r   r   r   r   r   r   marginrj  c                 S      t |||fi |S rG   )r   r   r   i1i2r   r   r2   r2   r8   r        z@module_inputs_torch_nn_CosineEmbeddingLoss.<locals>.reference_fnr   r   )r   r}   r2   r   r	   rQ   r|   ry   signrS   r   rW   r   rX   r[   r   r   r   r   r   r   r   r2   r2   r8   *module_inputs_torch_nn_CosineEmbeddingLoss  s*   




r  c                 K   s~   t t|||d}ttddt|ddd dttddt|dd	d
tt t|ddtdttddt|ddd
gS )Nr   r   alphar   r   r   c                 S   s   t |dk|d| d  S )Nr   r   rP   rg   wherer   r   r2   r2   r8   r     r   z,module_inputs_torch_nn_ELU.<locals>.<lambda>r   r2   scalarr7  r   r   r}   r   r   r   r   4d_inputr9  r:  r2   r2   r8   module_inputs_torch_nn_ELU  s&   






r  c                 K   sn   t t|||d}ttddt|ddd dttddt|dd	d d
dttddt|ddtdgS )Nr   r   r  r  c                 S   "   t |dk|dd|  d  S Nr   r   r   rP   r  r   r2   r2   r8   r        " z-module_inputs_torch_nn_CELU.<locals>.<lambda>r   r2   c                 S   r  r  r  r   r2   r2   r8   r     r  r  r/  r  r   r}   r9  r:  r2   r2   r8   module_inputs_torch_nn_CELU  s    





r  c                 K   sX   t t|||d}tt t|ddttdt|dddtt t|dd	td
gS )Nr   )r   r3  r4  rP   r   r3  rW  r   r7  r,  r   r}   r9  r:  r2   r2   r8   module_inputs_torch_nn_GLU  s   


r  c                 K   sf   t t|||d}ttdt|ddd ddttdt|dd	d d
tt t|ddtdgS )Nr   r   r2   c                 W       |d dt |td   S Nr         ?r   rg   erfr   sqrtr   r   xrV   r2   r2   r8   r   +       z-module_inputs_torch_nn_GELU.<locals>.<lambda>r  r/  r  c                 W   r  r  r  r  r2   r2   r8   r   /  r  r   r  r   r}   r9  r:  r2   r2   r8   module_inputs_torch_nn_GELU%  s    


r  c                 K   n   t t|||d}tt t|dddtt t|dtddtt t|dd	dtt t|d
ddgS Nr   r2   r  r7  r   r   r/  r  Zchannels_last_mem_format)r   r   r   r   r   Zchannels_last_3d_mem_formatr9  r:  r2   r2   r8   module_inputs_torch_nn_ReLU6  &   



r  c                 K   r  r  r9  r:  r2   r2   r8   module_inputs_torch_nn_ReLU6I  r  r  c              	   K   s   t t|||d}tt t|ddtt t|dtddttdt|ddd	ttd
t|ddd	ttdt|ddd	gS )Nr   r  r4  r   r   r/  r   Zwith_negvalr7  r   r   Zwith_zero_negvalr2   Zwith_negval_scalarr9  r:  r2   r2   r8    module_inputs_torch_nn_LeakyReLU\  s,   




r  c                 K   s   t t|||d}tt t|dddtt t|dtddtt t|dd	d
 ddttdt|ddd
 ddtt t|ddd
 ddttdt|ddd
 ddtt t|ddd
 ddttdt|ddd
 ddgS )Nr   r2   r  r7  r   r   r/  r   r   r   c                 S   (   t j|ddt j|dd|d d   S Nr   min)maxrg   clampr   r2   r2   r8   r   ~     ( z.module_inputs_torch_nn_PReLU.<locals>.<lambda>1dr   c                 S   r  r  r  r   r2   r2   r8   r     r  Z1d_multiparamr  c                 S   r  r  r  r   r2   r2   r8   r     r  Z2dc                 S   r  r  r  r   r2   r2   r8   r     r  Z2d_multiparam)r   r   r   r   r3  c                 S   r  r  r  r   r2   r2   r8   r     r  Z3dc                 S   r  r  r  r   r2   r2   r8   r     r  Z3d_multiparamr9  r:  r2   r2   r8   module_inputs_torch_nn_PReLUq  sR   







r  c                 K   sV   t t|||d}tt t|ddtt t|dtddtt t|ddd	gS )
Nr   r  r4  r   r   r/  r2   r  r7  r9  r:  r2   r2   r8   module_inputs_torch_nn_SELU  s   


r  c                 K   sb   t t|||d}tt t|ddd ddtt t|dtddtt t|d	d
d dgS )Nr   r2   c                 W      |t | S rG   rg   sigmoidr  r2   r2   r8   r         z-module_inputs_torch_nn_SiLU.<locals>.<lambda>r  r/  r   r   r  c                 W   r  rG   r  r  r2   r2   r8   r     r  r   r9  r:  r2   r2   r8   module_inputs_torch_nn_SiLU  s    


r  c                 K   sh   t t|||d}ttdt|ddd dttdt|dd	d d
dttdt|dtddgS )Nr   rP   r      c                 S   s&   t |t |ddddS NrP   Tr   r  )rg   r   divr   r   r   r2   r2   r8   r     s   & z0module_inputs_torch_nn_Softmax.<locals>.<lambda>r   r   r2   c                 S      t |t |ddS )Nr   Trg   r   r  r   r   r2   r2   r8   r     r   r  r/  r   r   r   r   r9  r:  r2   r2   r8   module_inputs_torch_nn_Softmax  s    


r  c                 K   sF   t t|||d}tt t|ddd dtt t|dtddgS )	Nr   rP   r   r   r  c                 S   r  NrP   Fr  r   r2   r2   r8   r     r   z2module_inputs_torch_nn_Softmax2d.<locals>.<lambda>r   rX  r   r/  r9  r:  r2   r2   r8    module_inputs_torch_nn_Softmax2d  s   

r  c              	   K   s   t t|||d}ttdt|ddd dttdt|ddd d	d
ttdt|ddd dd
ttdt|dtdd
gS )Nr   rP   r  c                 S   s*   t |t |dddd S r  )rg   r   div_r   r   log_r   r2   r2   r8   r     r   z3module_inputs_torch_nn_LogSoftmax.<locals>.<lambda>r   r  c                 S   "   t |t |dd S r  rg   r   r  r   r  r   r2   r2   r8   r     r  Z
multiparamr/  r   r2   c                 S   r  )Nr   Fr  r   r2   r2   r8   r     r  Zmultiparam_scalarr   r  r   r9  r:  r2   r2   r8   !module_inputs_torch_nn_LogSoftmax  s*   



r  c              	   K   st   t t|||d}ttdt|ddttdt|dddttdt|d	d
dttdt|dtddgS )Nr   rP   r  r4  )r   r   r   r   Zmultidimr7  r   r2   r  r   )r   r   r   r   r/  r9  r:  r2   r2   r8   module_inputs_torch_nn_Softmin  s$   



r  c              
   K   s   t t|||d}tt t|ddd dttdt|ddd dd	ttdd
t|ddd dd	ttdd
t|ddd dd	tt t|dtdd	gS )Nr   r  c                 S   s   t t |S rG   rg   log1pr   r   r2   r2   r8   r     s    z1module_inputs_torch_nn_Softplus.<locals>.<lambda>r   r   c                 S   s   dt t d|  S )Nr   r   r  r   r2   r2   r8   r     r>   betar/  c                 S   D   |d dk || |d dk |d d ttd|   S Nr   r  r  r   type_asrg   r  r   r   r2   r2   r8   r   	      ,Zbeta_thresholdr2   c                 S   r  r  r  r   r2   r2   r8   r     r  Zbeta_threshold_scalarr   r   r9  r:  r2   r2   r8   module_inputs_torch_nn_Softplus  s4   






r  c              	   K   sp   t t|||d}tt t|ddttdt|dddttdt|dddtt t|d	td
dgS )Nr   r  r4  rP   lambdar7  r2   Zlambda_scalarr   r   r/  r9  r:  r2   r2   r8   !module_inputs_torch_nn_Softshrink  s$   



r  c                 K   b   t t|||d}tt t|ddd dtt t|ddd dd	tt t|d
tdd	gS )Nr   r  c                 S      | dt| S NrP   r  rg   r   r   r2   r2   r8   r   0  r   z1module_inputs_torch_nn_Softsign.<locals>.<lambda>r   r2   c                 S   r  r  r  r   r2   r2   r8   r   3  r   r  r/  r   r   r9  r:  r2   r2   r8   module_inputs_torch_nn_Softsign*      


r  c                 K   V   t t|||d}tt t|ddtt t|dddtt t|dtdd	gS 
Nr   r  r4  r2   r  r7  r   r   r/  r9  r:  r2   r2   r8   module_inputs_torch_nn_Tanh;     


r  c                 K   r  r  r9  r:  r2   r2   r8   !module_inputs_torch_nn_TanhshrinkK  r  r  c              	   K   s~   t t|||d}ttddt|dddttddt|dddttddt|d	d
dttddt|dtddgS )Nr   r   r  r  Zthreshold_valuer7  g      $@Zlarge_valuer2   Zthreshold_value_scalarr   r   r/  r9  r:  r2   r2   r8    module_inputs_torch_nn_ThresholdZ  s&   







r  c                 K   r  )Nr   r  c                 S      |t t| S rG   rg   tanhFZsoftplusr   r2   r2   r8   r   s  r   z-module_inputs_torch_nn_Mish.<locals>.<lambda>r   r2   c                 S   r  rG   r  r   r2   r2   r8   r   v  r   r  r/  r   r   r9  r:  r2   r2   r8   module_inputs_torch_nn_Mishm  r  r  c                 K   s^   t t|||d}tt t|d|ddd dtt t|d|ddd dd	gt| S )
Nr   r  c                 S   s$   d|   tdd t||D  S )Nr  c                 s   s$    | ]\}}||    V  qd S rG   )r   r   )r4   abr2   r2   r8   rp    s    zBmodule_inputs_torch_nn_L1Loss.<locals>.<lambda>.<locals>.<genexpr>)r  r   zipr   r   r   r   r2   r2   r8   r     s    
z/module_inputs_torch_nn_L1Loss.<locals>.<lambda>r   r2   c                 S   s   d|   ||    S )Nr  )r  r   r   r  r2   r2   r8   r     s    r  r/  )r   r	   r|   ry   r1  r:  r2   r2   r8   module_inputs_torch_nn_L1Loss~  s   	r  c              	   K   s   t t|||d}di fdddifdddifddd	ifg}g }|D ];\}	}
|
fd
d}|ttdi |
t|d|d|	|d |ttdi |
t|d|dd|	 |d q |S )Nr   r@   r   r   r   r   r   r   r   c                 S   r   rG   )r%   r   r2   r2   r8   r     r   z9module_inputs_torch_nn_SmoothL1Loss.<locals>.reference_fnr   r   r}   r2   r   r  rS   r   rW   r   rX   r[   r   r   r   r   r   r   r2   r2   r8   #module_inputs_torch_nn_SmoothL1Loss  s8   


r  c                 K   s  t t|||d}t t||dd}t t||dd}di fdddifdddifd	dd
ifdd|difg}	ddd}
g }|	D ]*\}}|ttdi |t|dddd|dd||t |
fi |d q<|d}|tt|dt|dddd|dd|dt |
|dd |S )Nr   Fr@   r   r   r   r   r   r   r   r   r   r   c                 S   s^   ||   d| d|      }|d ur|| }|dkr|S |dkr+| |  S | S )NrP   r   r   )r   r   r  r   r   r   r   r   r   r   r2   r2   r8   bce_loss_reference_fn  s   "z=module_inputs_torch_nn_BCELoss.<locals>.bce_loss_reference_fnr}  {Gz?Gz?lowhighr   r}   r2   )r   scalar_weightr   Nr   r	   rQ   r|   ry   gtto)rS   r   rW   r   rX   r[   r   r   r   r   r  r   r   r   r  r2   r2   r8   module_inputs_torch_nn_BCELoss  s@   





r  c                 K   s   t t|||d}t t||dd}t t||dd}di fdddifdddifd	dd
ifdd|difdd|difg}	ddd}
g }|	D ]*\}}|ttdi |t|dddd|dd||t |
fi |d qC|S )Nr   Fr@   r   r   r   r   r   r   r   r   r   r  Zscalar_weightsr2   c                 S   s~   | j dd}d| |||  | |   }|d ur)|| }|dkr/|S |dkr;| |  S | S )Nr   r  rP   r   r   )r  mul_Zadd_Zexp_r  r   r  )r   r   r   r   r   r   Zmax_valr   r2   r2   r8   bce_withlogitsloss_reference_fn  s   4zQmodule_inputs_torch_nn_BCEWithLogitsLoss.<locals>.bce_withlogitsloss_reference_fnr}  r  r  r  r   r}   r  r  )rS   r   rW   r   rX   r[   r   r   r   r   r  r   r   r   r2   r2   r8   (module_inputs_torch_nn_BCEWithLogitsLoss  s.   



	r  c                 K   s(  t t|||d}t t|tjdd}t t||dd}g d}	di fdd|difddd	ifd
d
difdd	ddfg}
g }t|	|
D ]P\}\}}||fdd}|ttd*d|i|t|d|ddddd| d| |d |ttd*d|i|t|d|ddddd| d| |d |ttd*d|i|t|d|ddddd| d| |d |ttd*d|i|t|d|d dddd!| d| |d |dd d u r|ttd*d|i|t|d"|d"j	d	d#d$| d| |d |ttd*d|i|t|d%|d%j	d	d#d&| d| |d |ttd*d|i|t|d'|d'j	d	d#d(| d| |d |ttd*d|i|t|d|dj	d	d#d)| d| |d |ttd*d|i|t|d|d*dddd+| d| t t
d,d-d q@|S ).Nr   F)r   r   r   r@   r   r   r  r   rP   label_smoothingg333333?Zignore_index_label_smoothing)r   r  c                 S   s   t ||fd|i|S )Nr   )r   )r   r   r   r   r   r   r2   r2   r8   r        z=module_inputs_torch_nn_CrossEntropyLoss.<locals>.reference_fnr   )r   r   r   r   )r   r   r   r   r   r  Z4d_rV   r}   )r   r   r   )r   r   r   r   r   r   Z2d_)r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   Z4d_prob_target_)r   r   r   Z3d_prob_target_)r   r   Z2d_prob_target_Zhigher_dim_prob_target_r2   r.  Tr-  )r   r	   rg   r   r   rQ   r|   ry   r   Zsoftmaxr   )rS   r   rW   r   rX   r[   r   r   r   Z
reductionsr   r   r   r   r   r   r2   r2   r8   'module_inputs_torch_nn_CrossEntropyLoss
  s   


r  c                 K   s  t t|||d}t t|dd}di fdddifddd	ifd
ddifdddifg}tjtjg}	g }
t|	|D ]\}\}}|fdd}|dd}|dkrMdnd}|dkrUdnd}|
tt	d i |t	|d
d|d|||ddd| d|d |
tt	d i |t	|d
d|d|||dtjd|dtjd|d| d|d |
tt	d i |t	|d
d|d|||ddd| d|d |
tt	d i |t	|d
d|d|||dtjd|dtjd|d| d|d q5|
S )!Nr   F)r   r   r@   r   r   r   r   r   r   r   blank   c                 S   s   t ||||fi |S rG   )r   )r   r   r   r   iltlr   r2   r2   r8   r   r  r  z4module_inputs_torch_nn_CTCLoss.<locals>.reference_fnr   rP   r   )2   r   r   r   )r      )rW   r  r  )r  r  r  )r     r  Z_lengths_intlistsr}   r   Z_lengths_tensors)K   Z_1d_target_lengths_intlistsZ_1d_target_lengths_tensorsr2   )r   r	   rg   intr   r   r   rQ   r|   ry   r   tensor)rS   r   rW   r   rX   r[   r   r   r   Ztarget_dtypesr   Ztarget_dtyper   r   r   r  r  r  r2   r2   r8   module_inputs_torch_nn_CTCLossc  s|   



	
	r  c                 K   s   t t|||d}ttdddt|dddttdddt|d	d
dttdddt|dddttddddt|dddttddddt|dddttdddt|dddttddddt|dddttddddt|dddgS )Nr   r   r3  r  )r   r3  r   Z	1d_affiner7  r\  )r   r\  Z1d_affine_GNrP   )   r3  Z1d_affine_large_batchr   Fr   r   r   Z1d_no_affine_INr   r   Z1d_no_affine_LN)r   r3  r   r   Z	2d_affine)r   r   r   r   Z2d_no_affine_INZ2d_no_affine_LNr   r:  r2   r2   r8    module_inputs_torch_nn_GroupNorm  sT   











r  c                 K   sZ   t t|||d}ttdt|ddttdt|dddtt t|dtd	d
gS )Nr   r   )r   r   r   r   r4  r2   r  r7  r   r   r/  r9  r:  r2   r2   r8   !module_inputs_torch_nn_Hardshrink  s"   


r  c                 K   sB   t t|||d}tt t|dtddtt t|dddgS )Nr   r   r   r/  r  r  r7  r9  r:  r2   r2   r8    module_inputs_torch_nn_Hardswish  s   

r  c                 K   r  )Nr   r  c                 S      | ddS Nr   rP   r  r   r2   r2   r8   r         z1module_inputs_torch_nn_Hardtanh.<locals>.<lambda>r   r2   c                 S   r  r  r  r   r2   r2   r8   r     r  r  r/  r   r   r9  r:  r2   r2   r8   module_inputs_torch_nn_Hardtanh  s&   


r  c              
   K   s  t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifg}g }	|D ]S\}
}|fdd}|	ttdi |t|d|dd|dd|
|d |	ttdi |t|d|dd|ddd|
 |d q-|	S )Nr   Fr@   r   r   r   r   r   r   r   rw  r   c                 S   r   rG   )r   r   r2   r2   r8   r     r   z?module_inputs_torch_nn_HingeEmbeddingLoss.<locals>.reference_fnr  r   r   rP   r}   r2   r   )	r   r	   rQ   r|   ry   r  r  r  Zsub_r  r2   r2   r8   )module_inputs_torch_nn_HingeEmbeddingLoss	  s<   



r  c              	   K   s   t t|||d}di fdddifdddifddd	ifg}g }|D ]!\}	}
|
fd
d}|ttdi |
t|d|d|	|d q |S )Nr   r@   r   r   r   r   r   r   r   c                 S   r   rG   )r   r   r2   r2   r8   r   8  r   z6module_inputs_torch_nn_HuberLoss.<locals>.reference_fnr  r}   r2   r  r  r2   r2   r8    module_inputs_torch_nn_HuberLoss,  s&   


r  c              
   K   s  t t|||d}|dd}|d }d\}	}
}}}dddd	}|| }d
| }t|r/t|
|nt|	|
|t||dt|rEt|
|||nt|	|
|||t||ddt|r\t|
|nt|	|
|t||tddt|rtt|
|||nt|	|
|||t||tddgS )Nr   r   Frl  )r   r  ra  FT)r   r   r=  rF  rP   r   r   r,  r4  Ztracking_statsr7  Ztracking_stats_no_batch_dimr/  r   )r   r	   r   r|   ry   r   )rS   r   rW   r   rX   r[   r   r   rl  Znum_featuresr  rg  r_  Ztrack_running_statsZinput_no_batch_shape_dictrs  rr  r2   r2   r8   %module_inputs_torch_nn_InstanceNormNdF  s>   



r  c                 K   s   t t|||d}ttdgdt|dddttdgdt|dddttdgdd	t|dd
dttg ddt|dddttg ddd	t|dddttdgdt|dddttg dddd	dt|dddgS )Nr   r   r  r
  1d_elementwise_affiner7     r   r   !1d_elementwise_affine_large_batchF1d_no_elementwise_affiner   r   r   r   r   r   r   3d_elementwise_affine3d_no_elementwise_affiner   r   1d_empty_elementwise_affineT)Zelementwise_affiner   Z3d_elementwise_affine_no_biasr   r:  r2   r2   r8    module_inputs_torch_nn_LayerNormm  sJ   









r"  c                 K   s   t t|||d}dd }ttdgdt|dd|dttdgdt|d	d
|dttdgddt|dd|dttg ddt|dd|dttg dddt|dd|dttdgdt|dd|dgS )Nr   c           	         s   | j }|d u rt|jj }|j | j}| j} fddtt|D }|	 }|t
|dj|dd| j   }|d urB||9 }||S )Nc                    s   g | ]} | d  qS )rP   r2   ro  ndimr2   r8   r9     r>   zQmodule_inputs_torch_nn_RMSNorm.<locals>.rms_norm_reference_fn.<locals>.<listcomp>r   T)r   Zkeepdim)r  rg   ZfinforW   r$  normalized_shaper   rt  rf   floatZrsqrtr
  r   r  )	r   r   r   r  r%  r   dimsZ
upcasted_ir   r2   r#  r8   rms_norm_reference_fn  s   $
z=module_inputs_torch_nn_RMSNorm.<locals>.rms_norm_reference_fnr   r  r
  r  r}   r  r  Fr  r  r  r  r  r   r!  r   )rS   r   rW   r   rX   r[   r   r(  r2   r2   r8   module_inputs_torch_nn_RMSNorm  sN   








r)  c                 K   sb   t t|||d}ttdt|dddttdt|dddttd	d
ddt|dddgS )Nr   r   )rP   r   rW  r  r7  r   )rP   r   rW  rW  Z2d_uneven_padrP   r  r   r   )rP   r   rW  rW  rW  Z3d_custom_paramsr   r:  r2   r2   r8   (module_inputs_torch_nn_LocalResponseNorm  s"   


r*  c                 K   sf   t t|||d}ttddt|dddttdddt|ddttdddt|d	td
dgS )Nr         ?r   )rP   r   rW  normr7  r   r4  )r   rW  r   r/  r9  r:  r2   r2   r8   module_inputs_torch_nn_LPPool1d  s"   




r-  c                 K   f   t t|||d}ttdddt|ddttdddt|dtddttddt|dd	d
gS )Nr   r   rP   r   rW  rW  r4  r   rW  rW  r   r/  r+  r,  r7  r9  r:  r2   r2   r8   module_inputs_torch_nn_LPPool2d  "   




r1  c                 K   r.  )Nr   r   )rP   r   rW  rW  rW  r4  )r   rW  rW  rW  r   r/  r+  r,  r7  r9  r:  r2   r2   r8   module_inputs_torch_nn_LPPool3d  r2  r3  c                 K   sb   t t|||d}ttdt|dddttddt|dddttdddt|dd	dgS )
Nr   r   )r   r   r   r`  r7  r6  Treturn_indicesr5  r   r:  r2   r2   r8    module_inputs_torch_nn_MaxPool1d  s"   



r6  c              	   K   sl   t t|||d}ttdddt|dddttdddt|dd	dttdddd
dt|dddgS )Nr   r   r   r<  r?  r0  r`  r7  r/  r  Tr4  r5  r   r:  r2   r2   r8    module_inputs_torch_nn_MaxPool2d"  s"   




r8  c              
   K   s   t t|||d}ttdt|ddttddt|dddttdddt|dd	dttdddd
dt|dddgS )Nr   rE  rH  r4  r   r6  r7  rI  Zstride_paddingTr4  r5  r   r:  r2   r2   r8    module_inputs_torch_nn_MaxPool3d4  s*   




r9  c              
      s   t t ||d} fdd}ttdd| dt|ddd	ttd
d| dt|ddd	ttdd| ddt|ddd	ttdd| dt|dtddttd
d| dt|dtddgS )Nr   c                         t jdt j d S )N)rP   r   r   rW   r   rg   r   doubler   r2   r   r2   r8   make_random_samplesM  r  zGmodule_inputs_torch_nn_FractionalMaxPool2d.<locals>.make_random_samplesr   r   output_ratio_random_samples)rP   r   r   rW  ratior7  r  )r   r   Zoutput_sizerA  )rP   r   rW  r3  sizeTr@  rA  r5  ratio_return_indices)r   r   rW  ratio_no_batch_dimr/  )r   rW  r3  size_no_batch_dimr9  rS   r   rW   r   rX   r[   r   r>  r2   r   r8   *module_inputs_torch_nn_FractionalMaxPool2dJ  s@   





rJ  c                    s   t t ||d} fdd}ttdd| dt|ddd	ttd
d| dt|ddd	ttdd| dt|ddd	ttdd| ddt|ddd	ttdd| dt|dtddttd
d| dt|dtddgS )Nr   c                      r:  )N)r   r   r   r;  r<  r2   r   r2   r8   r>  o  r  zGmodule_inputs_torch_nn_FractionalMaxPool3d.<locals>.make_random_samplesr   r   r?  )r   r   r   r   r   rB  r7  rE  )r   r   r   rC  )r   r   rW  rW  rW  rD  )r   r   r   )r   r   r   )r   r      rW  r   ZasymsizeTrE  rF  )r   r   r   r   rG  r/  )r   rW  rW  rW  rH  r9  rI  r2   r   r8   *module_inputs_torch_nn_FractionalMaxPool3dl  sJ   






rL  c                 K   r  r  r9  r:  r2   r2   r8   module_inputs_torch_nn_Sigmoid  s.   



rM  c                 K   sb   t t|||d}tt t|ddd ddtt t|ddd d	tt t|d
tddgS )Nr   r2   c                 S      |   S rG   r  r   r   r2   r2   r8   r     r  z3module_inputs_torch_nn_LogSigmoid.<locals>.<lambda>r  r/  r  c                 S   rN  rG   rO  r   r2   r2   r8   r     r  r   r   r   r9  r:  r2   r2   r8   !module_inputs_torch_nn_LogSigmoid  s&   


rP  c              
   K   s   t t|||d}t t|tjdd}di fdddifdddifd	dd
ifdddifg}g }	|D ]&\}
}|fdd}|	ttdi |t|d|d|d |
|d q.|	S )Nr   Fr@   r   r   r   r   r   r   r   rw  r   c                 S   rx  rG   )r    ry  r2   r2   r8   r     r|  z>module_inputs_torch_nn_MarginRankingLoss.<locals>.reference_fn)r  r}   r2   )r   r	   rg   r   rQ   r|   ry   r  r  r2   r2   r8   (module_inputs_torch_nn_MarginRankingLoss  s*   




rQ  c                 K   s   t t|||d}t t|tjdd}di fdddifdddifd	dd
ifg}g }	|D ]A\}
}|fdd}|	ttdi |t|d|ddddd|
 |d |	ttdi |t|d|dddd|
|d q)|	S )Nr   Fr@   r   r   r   r   r   r   r   c                 S   r   rG   )r"   r   r2   r2   r8   r     r   zAmodule_inputs_torch_nn_MultiLabelMarginLoss.<locals>.reference_fnr  r   r   r  Z1d_r}   r  r2   r   r	   rg   r   rQ   r|   ry   r  r2   r2   r8   +module_inputs_torch_nn_MultiLabelMarginLoss  s:   


rS  c                 K   s   t t|||d}t t|tjdd}t t||dd}di fdddifdddifd	dd
ifdddifdddifdd|difg}	g }
|	D ]$\}}|fdd}|
ttdi |t|d|dddd||d qB|
S )Nr   Fr@   r   r   r   r   r   r   r   r   r   rw  r   r   r   r   c                 S   r   rG   )r!   r   r2   r2   r8   r   	  r   z<module_inputs_torch_nn_MultiMarginLoss.<locals>.reference_fnr  r   r   r  r}   r2   rR  )rS   r   rW   r   rX   r[   r   r   r   r   r   r   r   r   r2   r2   r8   &module_inputs_torch_nn_MultiMarginLoss	  s0   





rT  c                 K   s   t t|||d}t t|tjdd}t t||dd}di fdddifdddifd	dd
ifdd|difg}	ddd}
g }|	D ]$\}}|ttdi |t|d|dddd|t |
fi |d q=|S )Nr   Fr@   r   r   r   r   r   r   r   r   r   c                 S   sx   ||    d| |      }|d ur||9 }| | d |d }|dkr0|S |dkr8| S | S )NrP   r   r   r   )r  r   r   r   rD  r   r  r2   r2   r8   &multilabelsoftmargin_loss_reference_fn1	  s   &z_module_inputs_torch_nn_MultiLabelSoftMarginLoss.<locals>.multilabelsoftmargin_loss_reference_fnr  r   r   r  r}   r  r2   rR  )rS   r   rW   r   rX   r[   r   r   r   r   rU  r   r   r   r2   r2   r8   /module_inputs_torch_nn_MultiLabelSoftMarginLoss$	  s,   



rV  c              	   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}g }	|D ]#\}
}|fdd}|	ttdi |t|d|d |
|d q(|	S )Nr   Fr@   r   r   r   r   r   r   r   c                 S   r   rG   )r&   r   r2   r2   r8   r   X	  r   z;module_inputs_torch_nn_SoftMarginLoss.<locals>.reference_fn)r   r   r}   r2   r~  r  r2   r2   r8   %module_inputs_torch_nn_SoftMarginLossK	  s(   



rW  c                 K   s   g }t d ||||D ]@}|jj|jj}}	||	d< ||	d< tjj|i |	}
d}|j}d|jv r<|jd |jd< |jd= |t	t
|
|||jd q
|S )Nr   rW   r   src_maskmaskr7  ).module_inputs_torch_nn_TransformerEncoderLayerr~   rZ   r[   rg   r   TransformerEncoderLayerr   rQ   r|   ry   r   )rS   r   rW   r   rX   r[   samplesZlayer_module_inputZl_argsZl_kwargsZencoder_layer
num_layersr   r2   r2   r8   )module_inputs_torch_nn_TransformerEncoderf	  s*   


r^  c                 K   s  t t|||d}ttddddt|dddttddd	dtjt|dd
dttddd	dddt|dddg}d tjg d|tjdf}d tjg d|tjd	df}	t
|	|dddD ],\}
}}}}|ttddd	d|||dt|d|
|dt t|ddidd| d qadd }|rt
ddD ]&\}}|ttddd	dd||dt|d|r|nd d| d| d q|S ) Nr   r   r   rK  r   r  relu_activationr7  r   gelu_activationFr   r   FFTr   rW   r7  TFd_modelnheaddim_feedforwarddropoutr  
norm_firstr   rQ  )rX  src_key_padding_maskrj  r   r  r  no_batch_dim_batch_first_r/  c                 _   sX   | j sJ | d t  | |i |}W d    n1 s w   Y  | d |S )NFT)rX   trainrg   Zno_grad)moduler   rZ   r[   r!  r2   r2   r8   fast_path_reference_fn	  s   



zNmodule_inputs_torch_nn_TransformerEncoderLayer.<locals>.fast_path_reference_fnT)rh  r  ri  r   Z	fastpath_Z_norm_first_)r   r	   r|   ry   r  gelurg   r  boolr   ru  r   rQ   r   )rS   r   rW   r   rX   r[   r   r\  key_padding_masks
attn_masksrX  rj  ri  r  r   ro  r2   r2   r8   rZ  	  st    

rZ  c                 K   s  t t|||d}ttddddt|d|dddttddd	dtjt|d|dd
dttddd	dddt|d|dddg}d tjg d|tjdf}d tjg d|tjd	df}	t
|	|dddD ]\}
}}}}|
}|}|ttddd	d|||dt|d|d|
|||dt t|ddddd| d |d|d}}|s|dd|dd}}|d ur|	ddfd \}}|ttddd	d|||dt|||
|||dd| d| d| d qj|S )Nr   r   r   rK  r   r  r_  r7  r   r`  Fr   r   ra  rb  r7  rc  rd  rQ  )tgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskr   )rv  rw  rk  rl  r/  rP   r   Znorm_first_Z_batch_first_Z_bias_)r   r	   r|   ry   r  rp  rg   r  rq  r   ru  r   rQ   r   	transpose)rS   r   rW   r   rX   r[   r   r\  rr  rs  rt  rv  ri  r   r  ru  rw  srctgtr2   r2   r8   .module_inputs_torch_nn_TransformerDecoderLayer	  s    r{  c                 K   sn  t t|||d}g }d tjg d|tjdf}d tjg d|tjddf}	t|	|dddD ]\}
}}}}|
fd \}}|fd \}}|t	t
dddd	d	d
|||d	t
|d|d||||dt t|ddddd| d |d|d}}|s|dd	}|dd	}|d ur|ddfd \}}|t	t
dddd	d	d
|||d	t
||||||dd q0|S )Nr   ra  rb  r7  rc  r   r   r   rP   r   )	re  rf  rg  Znum_encoder_layersZnum_decoder_layersrh  r  ri  r   rQ  )rt  rX  rv  rj  r   )rv  rj  rk  rl  r/  r  r   r4  )r   r	   rg   r  rq  r   ru  r   rQ   r|   ry   r   rx  )rS   r   rW   r   rX   r[   r   r\  rr  rs  rY  r"  ri  r   r  rX  rt  rj  rv  ry  rz  r2   r2   r8   "module_inputs_torch_nn_Transformer
  sX    
r|  c                 K   sf   t tj|tjdd}ttdddt|ddddttdddt|dd	dd
d	ddgS )NFr   r   r   )Znum_embeddingsZembedding_dimr   r4  rP   i   rW  Zdiscontiguousr7  )r   rg   r   r   r|   ry   Zrandom_r   )rS   r   rW   r   rX   r[   Z
make_emptyr2   r2   r8    module_inputs_torch_nn_Embedding:
  s   

r}  c                 K   s   t t|||d}g }d}d tjg d|tjdf}	d tjg d|tjddf}
t||||	|
}|D ]I\}}}}}|t	t
ddd|||dt
|d	|d	|d	||d
td |t	t
ddd|||dt
|d	|d	|d	||d
t tddd q4|S )Nr   rc  ra  rb  )r   r   r   r   T)Z	embed_dimZ	num_headsr  r   add_bias_kvadd_zero_attnr7  )r"  	attn_maskr   Fr  )r   r	   rg   r  rq  r   ru  r   rQ   r|   ry   r#  )rS   r   rW   r   rX   r[   r   r\  Z	bool_valsrr  rs  Zproductsr   r~  r  r"  r  r2   r2   r8   )module_inputs_torch_nn_MultiheadAttentionI
  s<    	

r  c           	   	   K   s   t t|||d}ttddt|d|dtdttddddt|d|dtdg}|dd}|rK|ttdddd	d
t|d|dtd |S )Nr   r   r   r   Tr   is_rnnFrelu)r   nonlinearity)r   r	   r|   ry   r   r   rQ   )	rS   r   rW   r   rX   r[   r   r\  r  r2   r2   r8   #module_inputs_torch_nn_RNN_GRU_Cellh
  s,   r  c                 K   sl   t t|||d}ttddt|d|d|dftdttddddt|d|d|dftdf}|S )Nr   r   r   r   Tr   )r   r	   r|   ry   r+  rS   r   rW   r   rX   r[   r   r\  r2   r2   r8   module_inputs_torch_nn_LSTMCell
  s   r  c                 C   s*   | j }| d t| |}|j| |S )NF)r   r   r   data)r&  Zbatch_sizesZrequired_gradseqr2   r2   r8   make_packed_sequence
  s
   

r  Fc                 K   s  t t|||d}|d }d}	d}
d}d}g }|r t|	|
||}nt|
||}|D ]}|r3|\}}}}n|\}}}ddd|||d}ddd|||d}|rT||d< ||d< |ttdi |t|d	t t|d
d |ttdi |t|d	||r}dnddft t|d
d |r|ttdi |tt|dt	ddgt t|d
d |ttdi |tt|dt	g dt t|d
d q(|S )Nr   r  )r  r  FTr   )
input_sizehidden_sizer]  r  r   bidirectionalr   r  r   r   r  r   r   )r   r   r   r   )r   r   r   )r   r   r   r   r   r2   )
r   r	   r   rQ   r|   ry   r(  r  rg   r  )rS   r   rW   r   rX   Zwith_packed_sequencer[   r   r  r  r   r  r  r\  prod_genrZ   nlr  b_fbidir	cons_argscons_args_hiddenr2   r2   r8   module_inputs_torch_nn_RNN_GRU
  sl   





r  c              
   K   s  t t|||d}d}d}d}	d}
g }t|||	|
}|D ]i}|\}}}}d}d|d||||d}d|d||||d}|ttdi |t|dt t|dd	 |d
krV|n|}||r]dnd|f||rfdnd|ff}|ttdi |t|d|t t|dd	 q|S )Nr   r  )r   r   r   r   )r  r  r]  	proj_sizer  r   r  r<  r  r   r   r   r  r2   )r   r	   r   rQ   r|   ry   r)  )rS   r   rW   r   rX   r[   r   r   r  r  Z
proj_sizesr\  r  rZ   r  r  r  r  r  r  r  Zh_outhxr2   r2   r8   module_inputs_torch_nn_LSTM
  sB   

(
	r  c                 K   B   t t|||d}ttdt|dtdttdt|ddgS )Nr   rP   r  r   rP   r   r  r4  r9  r:  r2   r2   r8   &module_inputs_torch_nn_ReflectionPad1d	     

r  c                 K   r  Nr   rP   rX  r   rP   r   r   r   r   r   r   r3  r4  r9  r:  r2   r2   r8   &module_inputs_torch_nn_ReflectionPad2d  r  r  c                 K   r  )Nr   rP   r  r   rP   r   rP   r   rP   r   )r   r   r   r   r   r4  r9  r:  r2   r2   r8   &module_inputs_torch_nn_ReflectionPad3d'  r  r  c                 K   r  Nr   rP   rQ  r   r  rX  r4  r9  r:  r2   r2   r8   'module_inputs_torch_nn_ReplicationPad1d6  r  r  c                 K   r  r  r9  r:  r2   r2   r8   'module_inputs_torch_nn_ReplicationPad2dE  r  r  c                 K   r  )Nr   rP   r  r   rP   r   r   r   r   r3  )r   r   r   r3  rW  r4  r9  r:  r2   r2   r8   'module_inputs_torch_nn_ReplicationPad3dT  r  r  c                 K   r  r  r9  r:  r2   r2   r8    module_inputs_torch_nn_ZeroPad1dc  r  r  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rP   r  r   r  r4  r9  r:  r2   r2   r8    module_inputs_torch_nn_ZeroPad2dr  r  r  c                 K   r  )Nr   rP   r  r   r  )rP   r   r   r   r   r4  r9  r:  r2   r2   r8    module_inputs_torch_nn_ZeroPad3d  r  r  c                 K   F   t t|||d}ttddt|dtdttddt|dd	gS )
Nr   rP   r   rQ  r   r  r   rX  r4  r9  r:  r2   r2   r8   $module_inputs_torch_nn_ConstantPad1d     

r  c                 K   sF   t t|||d}ttddt|dtdttddt|ddgS )	Nr   rP   r   rX  r   r  r   r4  r9  r:  r2   r2   r8   $module_inputs_torch_nn_ConstantPad2d  r  r  c                 K   r  )
Nr   rP   r   r  r   r  rW  )rP   r   rP   r   rP   r4  r9  r:  r2   r2   r8   $module_inputs_torch_nn_ConstantPad3d  r  r  c              	         t t|||d}dd  ttdt|dtdttdt|d fd	d
dttdt|d fdd
dttdt|d fdd
dgS )Nr   c              	   S   sJ   t j| dddd|d  df | | ddddd|d f gddS )z input:
                [[[0., 1., 2.],
                  [3., 4., 5.]]]
                pad: (1, 2)
                output:
                    [[[2., 0., 1., 2., 0., 1.],
                      [5., 3., 4., 5., 3., 4.]]]
            Nr   rP   r   r   rg   catr&  padr2   r2   r8   padding1d_circular_ref  s   J	zDmodule_inputs_torch_nn_CircularPad1d.<locals>.padding1d_circular_refrP   rQ  r   r  r  c                        || j S rG   rn  r   r  r2   r8   r     r  z6module_inputs_torch_nn_CircularPad1d.<locals>.<lambda>)r   rP   c                    r  rG   r  r   r  r2   r8   r     r  r7  c                    r  rG   r  r   r  r2   r8   r     r  r9  r:  r2   r  r8   $module_inputs_torch_nn_CircularPad1d  s.   






r  c              	      s   t t|||d}dd  ttdt|dtdttdt|d fd	d
dttdt|d fdd
dttdt|d fdd
dgS )Nr   c              
   S   s   t j| dddd|d  df | | ddddd|d f gdd} t j| dddddd|d  df | | ddddddd|d f gddS )aS  input:
                [[[[0., 1., 2],
                   [3., 4., 5.]]]]
                pad: (1, 2, 2, 1)
        output:
            [[[[2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.]]]]
        Nr   r   r   r   rP   r  r  r2   r2   r8   padding2d_circular_ref  s   JVzDmodule_inputs_torch_nn_CircularPad2d.<locals>.padding2d_circular_refrP   rX  r   )rP   r   r   rP   )rP   rP   r   r   c                    r  rG   r  r   r  r2   r8   r     r  z6module_inputs_torch_nn_CircularPad2d.<locals>.<lambda>)r   r   r   r   c                    r  rG   r  r   r  r2   r8   r     r  )r   r   r   rP   )rP   rP   r   r   c                    r  rG   r  r   r  r2   r8   r     r  r9  r:  r2   r  r8   $module_inputs_torch_nn_CircularPad2d  s.   






r  c              	      r  )Nr   c                 S   s  t j| dddd|d  df | | ddddd|d f gdd} t j| dddddd|d  df | | ddddddd|d f gdd} t j| dddddddd|d  df | | ddddddddd|d f gddS )	aN  input:
                [[[[[ 0.,  1.,  2.],
                    [ 3.,  4.,  5.]],
                   [[ 6.,  7.,  8.],
                    [ 9., 10., 11.]]]]]
            pad: (1, 2, 2, 1, 1, 2)
            output: [[[[[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]]]]]
        Nr   r   r   r   r   r   rP   r  r  r2   r2   r8   padding3d_circular_ref  s   J%VbzDmodule_inputs_torch_nn_CircularPad3d.<locals>.padding3d_circular_refrP   r  r   r  )rP   rP   r   r   r   c                    r  rG   r  r   r  r2   r8   r   A  r  z6module_inputs_torch_nn_CircularPad3d.<locals>.<lambda>)r   r   r   rP   rP   r   c                    r  rG   r  r   r  r2   r8   r   F  r  )r   r   r   rP   r   r   c                    r  rG   r  r   r  r2   r8   r   K  r  r9  r:  r2   r  r8   $module_inputs_torch_nn_CircularPad3d  s.   *






r  Z
TestModuleZ	test_gradcuda)	active_ifrc   Ztest_gradgradZtest_non_contiguous_tensors)r  ro   rc   )   rW  ZTestExpandedWeightModuleZtest_module)rc   Z
TestDecompZtest_rnn_decomp_modulec                 K   s@  t t|||d}tttddt|dd|dddtjtddtttddt|dd|dd	dtjtd
dtttddt|dd|dddtjtddtttddt|dd|dddddtjtddtttdddt|dd|dd	dtjtd
dtttdddt|dd|dd	dtjtd
dg}|S )Nr   r   r  r   r  r4  5input has inconsistent input_size: got 11 expected 10r   r   r      9hidden0 has inconsistent hidden_size: got 21, expected 20r   5Input batch size 3 doesn't match hidden0 batch size 5rP   z.Expected hidden to be 1D or 2D, got 4D insteadr  r  	r   r	   r   r|   ry   r   r   ra   
ValueErrorr  r2   r2   r8   )module_error_inputs_torch_nn_RNN_GRU_Cellt  sr   				
	
8r  c                 K   s  t t|||d}tttddt|dd|dd|ddfdtjtddtttddt|dd|dd	|dd	fdtjtd
dtttddt|dd|dd|ddfdtjtddtttddt|dd|dddd|ddddfdtjtddg}|S )Nr   r   r  r   r  r4  r  r  r  r  r   r  rP   z/Expected hx\[0\] to be 1D or 2D, got 4D insteadr  r  r2   r2   r8   %module_error_inputs_torch_nn_LSTMCell  sN   			&&r  c                 K   sD   t ttddddtjtddt ttddddtjtddg}|S )Nr   r   rP   )r~   z%hidden_size must be greater than zeror  z$num_layers must be greater than zero)r   r|   ry   r   r   r  )rS   r   rW   r   rX   r[   r\  r2   r2   r8   $module_error_inputs_torch_nn_RNN_GRU  s   r  c                 K   R   t t|||d}|dd}tt|rtddntdt|ddtjtdd	gS )
Nr   is_constantFrP   r   r  r4  z(expected 2D or 3D input \(got 4D input\)r  	r   r	   r   r   r|   ry   r   r   r  rS   r   rW   r   rX   r[   r   r  r2   r2   r8   "module_error_inputs_torch_nn_Pad1d     
r  c                 K   r  )
Nr   r  FrP   r   r  r4  z(expected 3D or 4D input \(got 2D input\)r  r  r  r2   r2   r8   "module_error_inputs_torch_nn_Pad2d  r  r  c                 K   r  )
Nr   r  FrP   r   r  r4  z(expected 4D or 5D input \(got 2D input\)r  r  r  r2   r2   r8   "module_error_inputs_torch_nn_Pad3d  r  r  r   )r   r   Ztest_memory_formatrX   )r  )r   r   r   zSkipped!)r   )r   r   r   )ro   rc   )r   r   r   TZTestEagerFusionModuleInfoZ,test_aot_autograd_symbolic_module_exhaustiveZ#test_aot_autograd_module_exhaustive)rR   r   r   Ztest_check_inplace)rc   ro   rP   )rl  r   i  )version)ro   g-C6?)r   r   r   r   r   r   r   iE  )rl  r   rm  Ztest_cpu_gpu_parityg{Gzt?)r   r   r   ro   r   r   )rc   ro   r  )r   ro   r   r   r   r   )ZatolZrtolZtest_forwardcpu)r   r   r   )include_halfinclude_bfloat16gQ?r  )r   ro   r   )r  r  )r   ro   r   Z#test_if_train_and_eval_modes_differZtest_save_load)r   r   r   )rl  )r   rR   r   ZTestModuleMPSZtest_non_contiguousZtest_factory_kwargs)rR   r   r   r   )rc   r  gMb`?)r   r   r   r   )r  )r   r   )rR   r   r   r   )rR   r   r   r   r   )r   r   r   	module_db)F(W  rg   Zunittestcopyr   enumr   	functoolsr   r   ru  r   r   r   Ztorch.nn.functionalr   Z
functionalr  Ztorch.nn.utils.rnnr   Ztorch.testingr	   Z#torch.testing._internal.common_cudar
   Z$torch.testing._internal.common_dtyper   r   r   Z*torch.testing._internal.common_device_typer   r   r   r   r   r   r   r   r   r   r   Z2torch.testing._internal.common_methods_invocationsr   Z!torch.testing._internal.common_nnr   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   Z$torch.testing._internal.common_utilsr(   r)   r*   r+   r,   r-   r.   typesr/   operatorrF   ZaoZqatZquantizableZ	quantizedr0   rH   __annotations__Module	ContainerZ	NLLLoss2dZ	MaxPool2dr1   rJ   typefrom_iterabler<   r?   r  strr7   r;   r5   r3   rw   rk   r   Znamespace_namerB   rx   ry   r|   r   r   r   r   r   r   r   r   r	  r  r   r#  r(  r)  r+  r1  r;  rD  rJ  rN  rU  rY  rZ  r[  r^  re  rh  rk  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r)  r*  r-  r1  r3  r6  r8  r9  rJ  rL  rM  rP  rQ  rS  rT  rV  rW  r^  rZ  r{  r|  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  ZexpectedFailurer&  Z#rnn_gru_lstm_module_info_decoratorsr  r  r  r  r  r  backendsr   Zis_availableZis_macos_or_newerZ_macos15_or_newerZAdaptiveAvgPool1dZAdaptiveAvgPool2d
itemgetterZAdaptiveAvgPool3dskipZAdaptiveMaxPool1dZAdaptiveMaxPool2dZAdaptiveMaxPool3dZ	AvgPool1dZ	AvgPool2dr   Z	AvgPool3dZBatchNorm1dZBatchNorm2dZBatchNorm3dZCELUZConv1dr   ZConv2dZfloat64ZConv3dZConvTranspose1dZchalfZConvTranspose2dZ	complex32Z	complex64Z
complex128ZConvTranspose3dZCosineEmbeddingLossZELUZFractionalMaxPool2dZFractionalMaxPool3dZL1LossZSmoothL1LossZ
LazyConv1dZ
LazyConv2dZ
LazyConv3dZLazyConvTranspose1dZLazyConvTranspose2dZLazyConvTranspose3dZLinearZBilinearZLPPool1dZLPPool2dZLPPool3dZ	MaxPool1dZ	MaxPool3dZ	KLDivLossZMSELossZMarginRankingLossZMultiLabelMarginLossZMultiMarginLossZSoftMarginLossZMultiLabelSoftMarginLossZNLLLossZGaussianNLLLossZPoissonNLLLossZHingeEmbeddingLossZ	HuberLossZBCELossZBCEWithLogitsLossZCrossEntropyLossZCTCLossZGELUZGLUZ	GroupNormZ
HardshrinkZ	HardswishZHardtanhZInstanceNorm1dZInstanceNorm2dZInstanceNorm3dZLocalResponseNormZ	LayerNormZRMSNormZTransformerEncoderr[  ZTransformerDecoderLayerZTransformerZMultiheadAttentionZ	EmbeddingZReLUZ	LeakyReLUZReLU6ZPReLUZRNNCellZGRUCellZLSTMCellZSigmoidZ
LogSigmoidZSiLUZSoftmaxZ	Softmax2dZ
LogSoftmaxZSoftminZSoftplusZ
SoftshrinkZSoftsignZTanhZ
TanhshrinkZ	ThresholdZMishZRNNZGRUZLSTMZReflectionPad1dZReflectionPad2dZReflectionPad3dZReplicationPad1dZReplicationPad2dZReplicationPad3dZSELUZ	ZeroPad1dZ	ZeroPad2dZ	ZeroPad3dZCircularPad1dZCircularPad2dZCircularPad3dZConstantPad1dZConstantPad2dZConstantPad3dr  r2   r2   r2   r8   <module>   s  
4@$






	




E	@(C0'.
4%#0*Y@'#'"3"&#'JC.>*%)I


#<+ 
 $,/
?H

Z

n

 
  




  



  7  I



  `




           $
    +    3    ;    A

    K



    
a



    
~     




      
*



      
G      
\      
b
	
      
r      
x
       
        
                
        
        
'


        
3        
9        
C        
M
        
V        
\
        
e        
j        
o        
u
        
~         
          
          
          
'
          
.          
1
          
?          
I


          
V          
Y          
`          
g          
t          
           
            

            
            
+            
6            
D            
K            
T

            
a            
d            
k            
r            
v            
z            
~

             
              
              
              
              
               
(              
.              
5              
;              
>

              
K

              
X              
_              
e              
k              
p              
x              
{               
                
                
                
                
#                
*                
-                
3                
9                
=                
A                
H                
K                
Q        