o
    Zh                     @   s  d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z	 ddl
ZddlmZmZmZ ddlmZmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= e	rddl>m?Z? d%ddZ@edeAdejBjCfddZDdd ZEG dd de5ZFG dd  d e=ZGG d!d" d"eGZHG d#d$ d$eGZIdS )&aQ  
This module implements variable tracking for PyTorch nn.Module instances during Dynamo tracing.

It provides specialized handling for different types of nn.Module instances through several key classes:

- NNModuleVariable: Handles instance-specific module tracing, specializing on module id() and placing
  parameters directly on the torch.fx.GraphModule. This creates one graph per module instance.

- UnspecializedNNModuleVariable: Provides class-level module tracing, treating nn.Modules like other
  user-defined objects and passing parameters as inputs to the FX graph. This creates one graph per
  module class.

- UnspecializedBuiltinNNModuleVariable: Specifically handles built-in PyTorch modules (e.g. nn.Linear)
  with appropriate optimizations.

- FSDPManagedNNModuleVariable: Special handling for FSDP-wrapped modules with modified guarding behavior
  and parameter handling.

The module integrates with Dynamo's broader tracing functionality to handle module method calls,
parameter access, hooks, and other nn.Module behaviors while maintaining proper scoping and guarding
of module state.
    N)contextmanagernullcontext)TYPE_CHECKING   )graph_break_hintstrace_rules	variables)raise_observed_exceptionunimplementedunimplemented_v2UnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceConstDictKeySourceDictGetItemSourceFSDPNNModuleSourceGetItemSourceNNModuleSource"UnspecializedBuiltinNNModuleSourceUnspecializedNNModuleSource)get_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsset_example_valueunpatched_nn_module_callunpatched_nn_module_call_impl   )typestrValueMutationNewVariableTracker)invoke_and_store_as_constant)LazyVariableTracker)SliceVariable)UserDefinedObjectVariable)InstructionTranslatortxr.   c                    sd   t |dr0 fdd t||\}} fdd|D } fdd| D }|||| dS dS )	a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    Z_initialize_hookc                    s   t | rt|  fdd| D  S t| tr! fdd|  D S t| tttfr6t|  fdd| D S t| tj	j
rCt| jS | S )Nc                 3       | ]} |V  qd S N .0elemconvert_to_faker2   P/var/www/auris/lib/python3.10/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>]       zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>c                       i | ]	\}}| |qS r2   r2   r4   kvr6   r2   r8   
<dictcomp>_       zCinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<dictcomp>c                 3   r0   r1   r2   r3   r6   r2   r8   r9   a   r:   )r   type
isinstancedictitemslisttuplesettorchfxZProxyr   node)xr7   r/   r2   r8   r7   [   s   
z/initialize_lazy_module.<locals>.convert_to_fakec                    s   g | ]} |qS r2   r2   )r4   argr6   r2   r8   
<listcomp>h       z*initialize_lazy_module.<locals>.<listcomp>c                    r;   r2   r2   r<   r6   r2   r8   r?   i   r@   z*initialize_lazy_module.<locals>.<dictcomp>N)hasattrr"   rD   Z_infer_parameters)r/   modargskwargs
proxy_argsproxy_kwargsZ	fake_argsZfake_kwargsr2   rL   r8   initialize_lazy_moduleQ   s   
rV   
module_keyrQ   c                 c   sp    |  }|j|d}|dkr|  d| n| } z||jf|j| < |d |j|< d V  W |j| = d S |j| = w )Nr   @r&   )name	num_callsget	__class__Znn_module_stack)rW   sourcer/   rQ   Zfully_qualified_namerZ   r2   r2   r8   record_nn_module_stackm   s   r^   c                 C   s~   | r=d|j v r.t|j d r.|j d }t| d}t|tju r$t|d}t|tj	 d S t| t
jtjdd d S d S )Nforward__func__)attr)__dict__callabler   rA   types
MethodTyper   
make_guardr   ZCLOSURE_MATCH	functoolspartialZNOT_PRESENT_IN_GENERIC_DICT)r]   rQ   ZfwdZforward_sourcer2   r2   r8   &guard_to_detect_forward_monkeypatchingz   s   


ri   c                       s   e Zd ZddddhejZdededejj	ddf fddZ
d	d
 Zdd Zdd Z		d,ddZdd ZdddeddfddZdd Zdd Zd,ddZdd  Zd,d!d"Z	#	$	%	&		d-d'd(Z	)d.	$	%	&		d- fd*d+Z  ZS )/NNModuleVariablemodule_typerW   valuenn_module_stack_sourcereturnNc                    s:   t  jdi | || _|| _|| _| jsJ | j| _d S )Nr2   )super__init__rk   rW   rl   r]   rm   )selfrk   rW   rl   rS   r\   r2   r8   rp      s   
zNNModuleVariable.__init__c                 C      | j p| jS r1   rm   r]   rq   r2   r2   r8   get_nn_module_stack_source      z+NNModuleVariable.get_nn_module_stack_sourcec                 C   
   || _ d S r1   rm   rq   r]   r2   r2   r8   set_nn_module_stack_source      
z+NNModuleVariable.set_nn_module_stack_sourcec                 C   s   | j S r1   )rk   ru   r2   r2   r8   python_type   s   zNNModuleVariable.python_typer/   r.   c                 O   s   d S r1   r2   )rq   r/   r]   submodZ	key_extraoptionsr2   r2   r8   _wrap_submodule   s   z NNModuleVariable._wrap_submodulec                 C   s   |j | j}t|tjjr7g }| D ] \}}tj	
|}|j j|| j|tt| j|d || q|S t|tjjtjjtjjfsKJ t|| jsPJ g }t|D ]\}}||j j|| j|tt| j|d qV|S Nr]   )outputget_submodulerW   rB   rH   nn
ModuleDictrD   r   ConstantVariablecreateregister_attr_or_moduler   r   r]   append
ModuleListParameterList
Sequentialr'   	enumerate)rq   r/   baseresultrY   r~   Zname_varidxr2   r2   r8   unpack_var_sequence   s<   
z$NNModuleVariable.unpack_var_sequencerY   r)   c                 C   s@   |j | j}t||}ttt| j|t	j
 tj|S r1   )r   r   rW   rP   r   r   r   r]   rf   r   ZHASATTRr   r   r   )rq   r/   rY   rQ   r   r2   r2   r8   call_obj_hasattr   s   
z!NNModuleVariable.call_obj_hasattrc                 C   s   |j | j}t|ddS )NZtrainingF)r   r   rW   getattrrq   r/   rQ   r2   r2   r8   is_training   s   zNNModuleVariable.is_trainingc                 C   s6   |j | j}t| |jjdkrtt| t	)zIRestart analysis treating this module as an UnspecializedNNModuleVariablerp   )
r   r   rW   r   tagf_codeco_nameZmark_class_dynamicrA   r   r   r2   r2   r8   convert_to_unspecialized   s
   
z)NNModuleVariable.convert_to_unspecializedc                 C   sd   |j | j}t|rtd |j j| |r(|j jj| |dd}t|t	j
 S t|d}||v S )Nz-NNModuleVariable with custom __getattribute__T)Z
deleted_okrb   )r   r   rW   r!   r
   side_effectshas_pending_mutation_of_attrZ	load_attrrB   r   DeletedVariableobject__getattribute__)rq   r/   keyr   Zmutated_attr	base_dictr2   r2   r8   has_key_in_generic_dict   s   z(NNModuleVariable.has_key_in_generic_dictc                 C   sr   t |rtd t|dd}|du rdS t|tjstd dt|di}tj|| fi |	|tj
|gi S )zDCheck for a __getattr__ and handle it specially if it is implementedz6torch.nn.Module with a custom __getattribute__ definedT)Zignore_nn_module_getattrNz6torch.nn.Module with a non-function custom __getattr__r]   __getattr__)r!   r
   r   rB   rd   FunctionTyper   r   UserMethodVariablecall_functionr   r   )rq   r   r/   rY   
obj_sourceZ
getattr_fnr   r2   r2   r8   _custom_getattr_fallback  s   z)NNModuleVariable._custom_getattr_fallbackc                 C   s  | j ot| j |}|j| j}t|d}d}t }t	|j
D ]
}||j  q!| j s3td |dkr?tj| ||dS ||v rH|| }	nbd|v r]||d v r]||vr]|d | }	nMd|v rn||d v rn|d | }	n<d|v r||d v r|d | }	n+z
t||}	d}W n  ty   | j|||| j d	}
|
d ur|
 Y S tt| Y nw |d
krt| j | |dkr|stj|j
|dS |rt||	t|}t|ttfr|t|  | |S t|	t r| j rtt| j d|}t|d}tj!|	j"|d#|| gi S t|	t$rtj%|	j&t't(||dS t|	t)r)tj!|	*||dS t|	t+j,r8tj%|	| |dS t-|	sBt.|	rKt||	t|S t/ddt0| d| dt0|	 dt0|	 dd| dt0|	 dt0| ddgt1j2d tj| ||dS )Nrb   TzGETATTR with no sourcer   _modules_parameters_buffersF)r   r/   rY   r   r_   r\   fgetz$Unsupported nn.Module attribute typeznn.Module subclass: z, name: z, attribute type: z>Dynamo does not support tracing nn.Module attributes of type ``zRefactor your code so that `z	` (type `z`) is not an attribute of `zqCurrently supported attribute types are methods, classmethods, staticmethods, properties, constants, and tensors.)Zgb_typecontextZexplanationhints)3r]   r   r   r   rW   r   r   rG   inspectgetmror\   updaterb   keysr
   r   ZGetAttrVariablegetattr_staticAttributeErrorr   r	   ri   ZUserDefinedClassVariabler)   buildr   rB   rj   UnspecializedNNModuleVariabler{   rv   r   propertyUserFunctionVariabler   r   classmethodr   r`   r-   rA   staticmethod__get__rd   r   r   r   r   r'   r   ZSUPPORTABLE)rq   r/   rY   r]   r   r   Zobject_memberZall_class_attribute_namesrK   Zsubobjr   outr2   r2   r8   var_getattr  s   



zNNModuleVariable.var_getattrrR   list[VariableTracker]rS   dict[str, VariableTracker]c                 C   s&  |j | j}t| j|  || t|}t|tjj	rj|j
jtjj	ju rjt|r/| | |r5J d|r9J |\}|j D ]\}}||j j|| j|tt| j|d|gi  | }qA|W  d    S |r||jd uru|j| _t|||| |j  r|jdr|jdkrt|dddr| | ddlm}	 |	||j jd	| jgt||R  d
W  d    S | jsJ dt|tj j!r|j}
t| jd}n	|j"}
t| jd}t#|
t$j%r|
j&}
t|d}| g| }nt#|
t$j'sJ |(t)j*|
|d||W  d    S 1 sw   Y  d S )Nz3Expected lazy sequential isn't a valid combination?r   )z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)Zcheck_forward_hooksZcheck_backward_hooksr&   wrap_fx_proxyZcall_moduler/   proxyzsMust provide a valid source in order to inline, since inlined function may have default args which must be guarded.r_   
_call_implr`   )+r   r   rW   r^   rv   r   rB   rH   r   r   r\   r_   r    r   r   rD   r   r   r   r   r]   popcls_to_becomerk   rV   Zis_root_tracer
__module__
startswithbuilderr   create_proxyr"   rI   GraphModuler   r   rd   re   r`   r   inline_user_function_returnr   r   )rq   r/   rR   rS   rQ   Zis_lazyrM   Z
child_namer~   r   fn	fn_sourcer2   r2   r8   r   |  s   


#




J
&zNNModuleVariable.call_functionFc                    s  ddl m mm 
jj
fdd}	dv r(
S 	dkrJt
j
	  |	W  d    S 1 sEw   Y  	dkr]t
tjjr] dS 	d	krd  siJ td
 srJ d
 jd j }t|tr|S |jj}jj|tt
jdS |rt	}	jj d	 d	t|		S 	fdd	fdd}

fdd} 
fdd}dd 	dkrjjt
jd   srJ g }! D ]\	}|"|	| q|t# dS 	dkrFjjt
jd   g }j$d7i |
ddD ]\	}|"|	| q1|t# dS 	dkr{jjt
jd    g }j%d7i |
ddd!D ]\	}|"|	| qf|t# dS 	d"krjjt
jd   g }j&d7i |
d#dd!D ]\	}|"|	| q|t# dS 	d$krЈjjt
jd   sȈrJ |! S 	d%krjjt
jd   |& S 	d&krjjt
jd   |j$d7i |
dS 	d'kr$jjt
jd    |j%d7i |
dS 	d(krJs/r1J g }' D ]	|" 	 q7|t# dS 	d)kr]sUrWJ | S 	d*krshrjJ g } D ]\	}|"|	| qp|t# dS 	d+krsrJ  t(S 	d,krtt)j*j+t)j*j,frrd
  r d
 - j.v S 	d-krst(dksJ t)j*j+j/t)j*j0j/t)j*j,j/t)j*j1j/t)j*j2j/f}t3j/|vr9td
 t4js J t5d
 d
 - tt6t7fsJ t	j8}	t|	t9j:sJ tt
j	d.};t4j<|	|d
gt= S 
js?J td
 t>rjj?rg }t=t@t(d
 -  }tAd
 -  D ]\}}|| tt
j}|"jj||d qfd
 -  }jj|
 d/tt
jd
 - d}|S 
B dd0lCmD} td
 |rd
 Ejnd
  rΈd
 - n	tFd1d
    }jj|
jtt
jdS 	d2kstt)j*jGjHjIr 	d3kstt)j*jGjHjJr-	d4kr-t	j8}	tt
j	d.};t4j<|	|d
g S 	jjKv rQtLjjK	 rQtMd5d6 tNOP D rQ|	S tQ R	S )8Nr&   )r   ListIteratorVariableTupleVariablec                    s^   j djdi }t|j t \}}ddlm} |j jd| |g|R |ddS )NZget_attrr2   r&   r   call_method)rR   rS   r   )r   r   rW   r#   rJ   r"   r   r   )rY   Z	mod_proxyrT   rU   r   )rR   rS   modulerq   r/   r2   r8   generic_call_method_helper  s$   
z@NNModuleVariable.call_method.<locals>.generic_call_method_helperr   Z_wrapped_call_implr_   _check_input_dimTZ_get_item_by_idxr   r   __resultc                      s4   t dd t  D std  d S d S )Nc                 s   s    | ]}|  V  qd S r1   )is_python_constantr4   rK   r2   r2   r8   r9   5  s    
zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>znon-const NNModule method )all	itertoolschainvaluesr
   r2   )rR   rS   rY   r2   r8   assert_all_args_kwargs_const4  s
   zBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_constc                     s^     t }t|jdd D i dd  D      j  fdd| D S )Nc                 S   s   g | ]}|  qS r2   as_python_constantr   r2   r2   r8   rN   >  rO   zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<listcomp>c                 S   s   i | ]	\}}||  qS r2   r   r<   r2   r2   r8   r?   ?  r@   zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<dictcomp>c                    s   i | ]}| | qS r2   r2   )r4   r=   Z
bound_argsr2   r8   r?   C  s    )r   r   	signaturebindrD   apply_defaults	arguments)namesr   )rR   r   rS   r   rY   r   r8   
get_kwargs:  s   

z0NNModuleVariable.call_method.<locals>.get_kwargsc                    sF   g }| D ]\}}| jj||tj|d q |t dS )Nr   Zmutation_type)r   r   r   r   r]   r(   )rD   r   rY   r~   )r   
gen_sourcer   rq   r/   r2   r8   wrap_valuesE  s   z1NNModuleVariable.call_method.<locals>.wrap_valuesc              
      s.     | jj|| tj| dgS r   )r   r   r   r   r]   )rY   obj)r   r   r   r   rq   r/   r2   r8   named_embedR  s   z1NNModuleVariable.call_method.<locals>.named_embedc                 S   sJ   | d}|d dkr| S t|dkr#|d}t| |} t|dks| S )N.r    )splitlenr   r   )r]   rY   Z
name_splitrK   r2   r2   r8   r   _  s   


z0NNModuleVariable.call_method.<locals>.gen_sourcenamed_childrenr   r   named_parametersr   prefixrecursenamed_buffersr   Zremove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r   rD   __len____contains____getitem__r`   z.__getitem__(slice))SymNodeVariablez%getitem on NNModuleVariable with key Z_get_abs_string_indexZ_conv_forwardZ_output_paddingc                 s   s    | ]	}t |tjV  qd S r1   )rB   r   TensorVariabler   r2   r2   r8   r9     s
    

z/NNModuleVariable.call_method.<locals>.<genexpr>r2   )Sr   r   r   r   rW   r   r   r   r^   rv   r   Zis_torch_inline_allowedr   getfiler\   r   r   r   rB   rD   rl   r   r   r   r   r]   r   __name__r*   guard_on_key_orderaddr   rY   r   r   r(   r   r   r   r   r   rH   r   r   ZParameterDictr   r   r   r   r   r   rA   r   r'   strintr`   rd   r   r   r   rE   r,   Zexportranger   r   Ztensorr   Zevaluate_exprr
   r   convZ_ConvNdZ_ConvTransposeNdrb   rc   r   r   r   r   ro   r   )rq   r/   rY   rR   rS   Zconstantr   Zmod_varr~   r   r   r   r   r   parambufferZbuiltin_supportedsrcr   r   Z
new_moduleZnew_module_variabler   r   rr   )r   r   r   rR   r   r   r   rS   r   rY   rq   r/   r8   r     s   



	















"




zNNModuleVariable.call_methodr/   r.   rR   r   rS   r   rn   r)   )F)r  r   __qualname__r)   _nonvar_fieldsrA   r  rH   r   Modulerp   rv   r{   r}   r   r   r   r   r   r   r   r   r   r   __classcell__r2   r2   rr   r8   rj      sf    

 



h
qrj   c                       s   e Zd ZdddhejZ	 d# fddZdd	 Zd
d Zdd Ze	e
ddd Z fddZ								d$ddZ						d% fddZd&ddZd& fdd Zd&d!d"Z  ZS )'r   
value_typeis_state_mutatedrm   rn   Nc                    sr   t |tjjju rtdd|v r%t|d dd }t ||u r%t ||d< t jdd|i| d| _	| j
| _d S )NzScriptModules aren't supported in UnspecializedNNModuleVariable becuase their .forward function isn't a static member of their typer  r   rl   Fr2   )rA   rH   Zjit_scriptZRecursiveScriptModuler   r   ro   rp   r  r]   rm   )rq   rl   rS   Zlazy_value_to_becomerr   r2   r8   rp   2  s   z&UnspecializedNNModuleVariable.__init__c                 C      t |ts	t|S |S r1   )rB   r   rq   Zattr_sourcer2   r2   r8   _wrap_sourceK     
z*UnspecializedNNModuleVariable._wrap_sourcec                 C   rs   r1   rt   ru   r2   r2   r8   rv   P  rw   z8UnspecializedNNModuleVariable.get_nn_module_stack_sourcec                 C   rx   r1   ry   rz   r2   r2   r8   r{   S  r|   z8UnspecializedNNModuleVariable.set_nn_module_stack_sourcec                      s0   t jjjt jjjh  fddt jjj D S )Nc                    s(   h | ]}t |d r| vrt|jqS )__code__)rP   idr  r   	supportedr2   r8   	<setcomp>[  s    zFUnspecializedNNModuleVariable._nn_module_method_ids.<locals>.<setcomp>)rH   r   r  __setattr__rp   rb   r   r2   r2   r  r8   _nn_module_method_idsV  s   
z3UnspecializedNNModuleVariable._nn_module_method_idsc              
      sz   z	t | jd}W n ty } zt|d }~ww |tjjjtjj	jtjj
jfv r7|t|| gi |S t |S )N__iter__)r   r   r  r   NotImplementedErrorrH   r   r   r!  r   r   r   r   r   r   ro   )rq   r/   r   err   r2   r8   r   a  s(   z1UnspecializedNNModuleVariable.unpack_var_sequencer/   r.   rR   r   rS   r   r)   c                 C   s  | j }t|r|jd ur|j| _t|||| t|tjjs-|j	j
tur-d}t| j|}nd}t| j|}t|j	tjrt|jtjr|j	j
tu r|jj
tu rd|jvrt|d}t|tjr|j}| |d  s| |d  s| |d  s| |d  s||d s||d	 s||d
 s||d sd}| jj}| jrtt| jd|}	nd }	t| j| | jrttt || ! ||nt" }
|
 t#j$||	d%|| gt&| |W  d    S 1 sw   Y  d S )N__call__r   r_   _backward_hooks_backward_pre_hooks_forward_hooks_forward_pre_hooksZ_global_backward_pre_hooksZ_global_backward_hooksZ_global_forward_hooksZ_global_forward_pre_hooksr\   r   )'rl   r   r   r  rV   rB   rH   rI   r   r$  r`   r$   r   r   rd   re   r   r%   rb   r   r   r   Znn_modules_globals_vtr   Zrealizer   r_   r]   r   ri   r^   r  r  rv   r   r   r   r   rE   )rq   r/   rR   rS   rQ   rY   r   Zforward_methodZ
globals_vtr]   ctxr2   r2   r8   r   y  sn   


$z+UnspecializedNNModuleVariable.call_functionc           
         s  |dv r*t | j|}| jrtt| jd|}nd }tj||d|| gt| |S |t | jdi vrzt	
t| j|}W n tyI   d }Y nw t|trhttt| jd|d}|tj|j|d||S t|dr}t|j|  v r}td|  |dkrd	| jjv r|d
  }|d }	t|	tjr|	 tjju s|| jjd	 v rd| _n || jjd v rd| _nt|	tjtjfs|| jjd v rd| _|tjj j!u rt|d tj"r|ttjj j#| |d
 g|S t$ %||||S )Nr   r\   r   rb   r`   r  z&UnspecializedNNModuleVariable missing r  r   r   r&   Tr   r   )&r   r  r]   r   r   r   r   rE   rl   r   r   rA   r   rB   r   r   r`   rP   r  r  r   r
   rb   r   r  r}   rH   r   	Parameterr  rj   r   r  r  r   __delattr__ro   r   )
rq   r/   rY   rR   rS   r   r]   methodZ	attr_namerl   rr   r2   r8   r     sn   




z)UnspecializedNNModuleVariable.call_methodc                 C   s&   |  ||}t|tjr||S d S r1   )r   rB   r   ConstDictVariableZmaybe_getitem_const)rq   r/   fieldname_vtZdict_vtr2   r2   r8   getattr_helper  s   
z,UnspecializedNNModuleVariable.getattr_helperc                    s
  |dv r3|j j| |s3t| j|}t|tr3t|dkr3| jr.t	| j|}t
|tj ti S | jr~|dv r~|j j| |s~t| j|}t	| j|t
tj |j j  fdd t fddt| D }tj|t|dS t ||S )	N)r%  r&  r'  r(  r   )r(  r'  c                    s4   t j|}t | }t |}t||}||fS r1   )r   r   r   r   r   r+   )ir=   r>   r   Z
source_keyZsource_valuerl   )hooks_dict_sourcer2   r8   build_key_valueF  s
   

zBUnspecializedNNModuleVariable.var_getattr.<locals>.build_key_valuec                 3   s$    | ]\}\}} |||V  qd S r1   r2   )r4   r1  r=   r>   )r3  r2   r8   r9   Q  s    
z<UnspecializedNNModuleVariable.var_getattr.<locals>.<genexpr>r   )r   r   r   r   rl   rB   rC   r   r]   r   r   rf   r   ZEMPTY_NN_MODULE_HOOKS_DICTr   r-  ZSEQUENCE_LENGTHr  r  rY   r   rD   ZNNModuleHooksDictVariablerA   ro   r   )rq   r/   rY   Z
hooks_dictZhooks_sourcer   rr   )r3  r2  r8   r     s>   


z)UnspecializedNNModuleVariable.var_getattrc                 C   sZ   t |}| |d|}|du r| |d|}|du r"| |d|}|du r+tt| |S )a  
        Dynamo tracing of nn.Module __getattr__ can be expensive if the model
        has deep submodule hierarchy. Since the __getattr__ is stable, we can
        directly look into the underlying datastructures. This saves a lot of
        compilation time.
        r   Nr   r   )r   r   r0  r	   r   )rq   r/   rY   r/  r   r2   r2   r8    manually_trace_nn_module_getattrZ  s   

z>UnspecializedNNModuleVariable.manually_trace_nn_module_getattrrn   N)r/   r.   rR   r   rS   r   rn   r)   r  r  )r  r   r  r-   r  rp   r  rv   r{   r   rg   	lru_cacher   r   r   r   r0  r   r4  r  r2   r2   rr   r8   r   "  sF    	
C
[=r   c                   @   s   e Zd ZdZdd ZdS )$UnspecializedBuiltinNNModuleVariablezg
    Differentiates between builtin nn modules (e.g. torch.nn.Linear) and user defined nn modules.
    c                 C   r  r1   )rB   r   r  r2   r2   r8   r  q  r  z1UnspecializedBuiltinNNModuleVariable._wrap_sourceN)r  r   r  __doc__r  r2   r2   r2   r8   r7  l  s    r7  c                       s*   e Zd ZdZd fddZdd Z  ZS )	FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    rn   Nc                    s<   | dd }|d usJ dt jdd|i| || _d S )Nr]   zMFSDPManagedNNModule depends on having an accurate source to control guarding.rl   r2   )r[   ro   rp   r]   )rq   rl   rS   r]   rr   r2   r8   rp     s   

z$FSDPManagedNNModuleVariable.__init__c                 C   s,   t |ttfstjjjrt|S t|S |S r1   )rB   r   r   rH   Z_dynamoconfigZskip_fsdp_guardsr  r2   r2   r8   r    s   
z(FSDPManagedNNModuleVariable._wrap_sourcer5  )r  r   r  r8  rp   r  r  r2   r2   rr   r8   r9  w  s    	r9  r  )Jr8  rg   r   r   rd   
contextlibr   r   typingr   Ztorch.nnrH   r   r   r   r   excr	   r
   r   r   r   Zguardsr   r   Zmutation_guardr   r]   r   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r   r'   r(   r)   Z	functionsr*   Zlazyr+   listsr,   Zuser_definedr-   Ztorch._dynamo.symbolic_convertr.   rV   r  r   r  r^   ri   rj   r   r7  r9  r2   r2   r2   r8   <module>   sH   (<

!       L