a
    hIA                     @   s  d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dl m!Z! erddl"m#Z# G dd de$Z%G dd de$Z&ee'dZ(dd Z)G dd de!Z*dS )a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)TYPE_CHECKING)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                   @   s   e Zd ZdS )ArgMappingExceptionN__name__
__module____qualname__ r   r   O/var/www/auris/lib/python3.9/site-packages/torch/_dynamo/variables/optimizer.pyr   7   s   r   c                   @   s   e Zd ZdS )GuardInstallExceptionNr   r   r   r   r   r   ;   s   r   Z
perf_hintsc                 C   sT   ddl m} | jrL|| jjd}tjj| d u}|rF|pD|j	
| S |S ndS d S )Nr   )get_managerFT)Ztorch._inductor.cudagraph_treesr   is_cudadeviceindextorchZ_dynamoutilsZget_static_address_typeZcurrent_nodeZ_is_cuda_graph_recorded_tensor)xr   managerZis_static_addressr   r   r   _is_static_for_cudagraphsB   s    
r'   c                       s   e Zd ZdddhejZd"dd fddZdd	d
d fddZdd fddZdd Zdd Z	dd Z
dd Zdd ZddddZddddZd d! Z  ZS )#OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesN)returnc                    s8   t  j|fi | |pi | _|p$i | _|p0t | _d S N)super__init__r)   r*   setr+   )selfvaluer)   r+   r*   kwargs	__class__r   r   r/   \   s    

zOptimizerVariable.__init__zlist[VariableTracker]zdict[str, VariableTracker]r   )argsr3   r,   c           
   
      s   |dkrz|  | |   | j|i |\}}| jj|i |}| | | ||||| dt| j }||| j | 	| t
|W S  ttfy }	 zW Y d}	~	n
d}	~	0 0 t ||||S )zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_groupZ__optimizer_N)graph_break_if_pending_mutationmove_step_if_cpuget_python_argsr2   r7   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r   r.   call_method)
r1   txnamer6   r3   py_args	py_kwargsZret_valZmangled_name_r4   r   r   rA   i   s    


zOptimizerVariable.call_methodr   )rB   c                    sn   |dv rt | |t| j|dS |dkr`ddlm} | jjD ]}|d D ]}|| qFq:| | t 	||S )N)r7   step)sourceparam_groupsr   mark_static_addressparams)
r   r   rH   
decoratorsrK   r2   rI   _set_capturabler.   var_getattr)r1   rB   rC   rK   grouppr4   r   r   rO      s    
zOptimizerVariable.var_getattrc                 C   s\   | j jD ]N}|d D ]@}|jj}|jt|d }|r||rddlm	} |dqqd S )NrL   r   )UnsupportedzPending mutation on parameter)
r2   rI   outputside_effectsZid_to_variablegetr=   Zhas_pending_mutationexcrR   )r1   rB   grQ   rT   variablerR   r   r   r   r8      s    z1OptimizerVariable.graph_break_if_pending_mutationc           	         s   ddl m}  fdd} jjD ]}||r d|d< q  joFt jd}|t| jj|}|j	D ]$}t
td}td|j	|< qfd S )Nr   LazyVariableTrackerc                    sJ   d}d}|  dg D ]$}||jp$|jM }|| jjvM }qd| v oH|oH|S )NTrL   
capturable)rU   r    Zis_xpur2   state)rP   Zall_uninitializedZall_gpurQ   r1   r   r   safe_to_set_capturable   s    zAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturableTr[   rI   ) rZ   r2   rI   rH   r   realize_allr   builditemsr   Z_HashableTrackerr   r@   )	r1   rB   rZ   r^   rP   rH   param_groups_vtZparam_group_vtkeyr   r]   r   rN      s    

z!OptimizerVariable._set_capturablec                    s<   fdd  fdd|D } fdd|  D }||fS )z9Get python values equivalent to the variable tracker argsc                    sp   t | tr|  S t | tr&| js&g S t | trht | jtrht | jjt	rh| jjj
dkrh jj| jj S td S )NrI   )
isinstancer   Zas_python_constantr   rb   r   rH   r   baser   memberr2   rI   r"   r   )argr]   r   r   map_arg   s    

z2OptimizerVariable.get_python_args.<locals>.map_argc                    s   g | ]} |qS r   r   ).0rh   ri   r   r   
<listcomp>       z5OptimizerVariable.get_python_args.<locals>.<listcomp>c                    s   i | ]\}}| |qS r   r   )rj   kvrk   r   r   
<dictcomp>   rm   z5OptimizerVariable.get_python_args.<locals>.<dictcomp>)rb   )r1   r6   r3   new_argsZ
new_kwargsr   )ri   r1   r   r:      s    z!OptimizerVariable.get_python_argsc                 C   s@   | j j D ].\}}d|v r|d jr|d |j|d< qd S )NrG   )r2   r\   rb   Zis_cputor!   )r1   rQ   r\   r   r   r   r9      s    z"OptimizerVariable.move_step_if_cpuc                    s  ddl m  ddlm} i | _i | _ fdd}ttj|| j	j
 | joRt| jd}|t|| j	j|}| jo|t| jd}t|| j	j
|}|  |jj| t| j	j|jD ]^\}}	t|d	 d
krL|d	 D ]p}
|
jd urd }t| j	j
 D ]\}}||
u r|} qq|r|t|| j	j
|
 t|t||  qLq|	|td	}d}g }tt|d	 | |D ]l\}\}}|j}|| j|< t!|d}|jd ur|| j|j< t"|jsd}|#| nt$|%t&j' q~|st()t*j+rdd |D }t(,d| qt| j	j
 D ]\}\}}t|t||}|jj| t| D ]J\}\}}t-|tjr^|| jvr^|| jvr^t|t||| j|< q^q(d S )Nr   rJ   r   rY   c                    s    |  d S r-   r   )r%   rJ   r   r   mark_static   s    zEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_staticrI   r\   rL   r   TgradFc                 S   s   g | ]}|  qS r   rC   )rj   srcr   r   r   rl   A  rm   zDOptimizerVariable.map_sources_and_install_guards.<locals>.<listcomp>)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.).rM   rK   ZlazyrZ   r)   r*   r   r#   Tensorr2   r\   rH   r   r`   r   ra   rI   ZrealizerS   Zguard_on_key_orderaddziprb   lenrt   	enumeratekeysr
   r	   Zgetitem_constr   r@   Zunpack_var_sequencer   r'   appendr   Z
make_guardr   ZCONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGwarningre   )r1   rB   rZ   rs   Zparams_groups_sourcerc   Zstate_sourceZstate_vtrP   Zgroup_vtparamZ	key_indexirn   Z	params_vtZ
all_staticZnon_static_gradsZp_indrQ   Zp_vtZparam_sourceZgrad_sourceidxr2   Zp_state_sourceZ	inner_idxro   r   rJ   r   r;      s    







z0OptimizerVariable.map_sources_and_install_guardsc                 C   s   ddl m} || jv rB|| | j| }| j|j|  nJ|| jv rX| j| }n4|| |	t
|}t|}| j|j|  t|||S )z%Wrap state tensor in a TensorVariabler   rJ   )rM   rK   r*   r+   rx   rS   Zmodule_key_namerC   r)   r>   r   r   r   ra   )r1   rB   Ztensor_valuerK   rH   Zglobal_namer   r   r   wrap_tensor\  s    


zOptimizerVariable.wrap_tensorc              	   C   s   t ||D ]\}}t|tr
t|ts.J dt|D ]`\}}	|jj| t|	tj	rn|j
| ||	 q6|jo~t|j|}
|j
t||	|
 q6q
dS )z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)ry   re   r   listr{   rS   rT   Zmutationr#   rw   rb   r}   r   rH   r   r   ra   )r1   rB   r6   r3   rD   rE   rh   Zpy_argr   valrH   r   r   r   r<   v  s    
z"OptimizerVariable.update_list_argsc                    s4   | j  | j|jj fdd}|j| d S )Nc                    s     fdd}t | d S )Nc                     sJ   D ]@}  j | d   j| d  jr4j  jrj  qd S r-   )_bufferspop_parametersZparams_flatclearZparams_flat_unwrap_subclassesru   )gmnames_to_deletetcr   r   clear_static_tensor_refs  s    
z\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs)weakreffinalize)r   r   r   r   r2   )r   r   init_finalizer  s    	z:OptimizerVariable.create_finalizer.<locals>.init_finalizer)r+   r2   rS   Ztracing_contextZadd_graph_finalizer)r1   rB   r   r   r   r   r?     s
    z"OptimizerVariable.create_finalizer)NNN)r   r   r   r   Z_nonvar_fieldsr/   rA   rO   r8   rN   r:   r9   r;   r   r<   r?   __classcell__r   r   r4   r   r(   T   s4   
   ! qr(   )+__doc__r   r   typingr   r#   Ztorch._loggingr   Ztorch.utils._pytreer   Zguardsr   r   rH   r   r	   r
   r   r   r   r$   r   rf   r   Zconstantr   Zdictsr   listsr   miscr   Zuser_definedr   Ztorch._dynamo.symbolic_convertr   	Exceptionr   r   r   r~   r'   r(   r   r   r   r   <module>   s,    
