a hy@sjUddlmZddlZddlmmZddlmZddlm Z gZ e e e d<ejjGdddZdS))OptionalN)Tensor)2_scripted_functional_optimizer_deprecation_warning__all__c @sdeZdZdeeeeeefeeeeeeed ddZee ed d d Z ee ed d dZ dS)_FunctionalAdamWMbP?g?g+?:0yE>{Gz?F) paramslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Cs$tddd|ks td|d|ks6td|d|dkrNdksbntd|dd|d krzdksntd |d d|kstd ||||d|d |d |_||_||_||_| |_tj t tj t t tj ffi|_ t|dkr| std d|i|_dS)N) stacklevelzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r rbeta1beta2rz%optimizer got an empty parameter listr )r ValueErrordefaultsrrrrtorchjitZannotatedictrstrstatelen param_group) selfr r r rrrrrrrr%V/var/www/auris/lib/python3.9/site-packages/torch/distributed/optim/functional_adamw.py__init__s2 $z_FunctionalAdamW.__init__)paramgradc Csvg}g}g}g}g}g}t|} |dur>||||||jvri|j|<|j|} td| d<tj|tjd| d<tj|tjd| d<|jrtj|tjd| d<|j|} || d|| d|jr|| d|| dtdt j |||||||j|j |j d|j d|j d |j d |j d |j |jdd| d Wdn1sh0YdS) NrstepZ memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrr rr rrrrr rrrrZ grad_scaleZ found_inf has_complex)r is_complexappendr!tensor zeros_likepreserve_formatrno_gradFadamwrrrr) r$r(r)params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr0r!r%r%r& step_paramGsf           z_FunctionalAdamW.step_param) gradientsc Cs|jd}g}g}g}g}g}g}t|t|krXtddt|ddt|d} t|jd|D]\} } | durl| t| O} || || | |jvri|j| <|j| } td| d<tj | tj d | d <tj | tj d | d <|j rtj | tj d | d <|j| } || d || d |j rL|| d || dqlt dt j|||||||j |j|jd |jd|jd|jd|jd|j|jdd| dWdn1s0YdS)Nr zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: Frr*r+r,r-r.rrr rrr/)r#r"rziprr1r2r!r3r4r5rr6r7r8rrrr) r$r@r r9r:r;r<r=r>r0r(Zgradientr!r%r%r&r*s|            z_FunctionalAdamW.stepN) rrr r FFFFF) __name__ __module__ __qualname__listrfloattupleboolr'rr?r*r%r%r%r&rs.  -=r)typingrrZtorch.optim._functionalZoptimZ _functionalr7rZ,torch.distributed.optim._deprecation_warningrrrEr __annotations__rscriptrr%r%r%r&s