a
    h]                    @   s  U d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZ ddlmZmZmZmZmZmZ ddlZddlZddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( dd	l)m*Z* dd
l+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA ddlBmCZCmDZD ddlEmFZF ddlGmHZHmIZI ddlJmKZK ddlLmMZM ddlNmOZOmPZPmQZQmRZRmSZS ddlTmUZU ddlVmWZWmXZXmYZY ddlZm[Z[m\Z\m]Z] ddl^m_Z_m`Z`maZambZbmcZc ddldmeZe ddlfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZmZmZmZ dd lmZmZ dd!lmZmZmZmZmZmZmZ dd"lmZmZmZmZmZmZ dd#lmZmZmZmZmZ dd$lmZmZmZmZmZmZmZmZmZmZmZmZmZ dd%lmZ dd&lmZ dd'lmZ dd(lmZmZmZmZmZmZmZmZmZ dd)lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ dd*lmZmZmZ dd+lmZ dd,lmZ dd-lmZ dd.lm Z mZmZmZmZmZ dd/lmZmZmZm	Z	 dd0l
mZmZmZ dd1lmZmZmZmZmZmZmZmZmZmZmZ zddlZW n ey
   dZY n0 er6dd2lmZ dd3lm Z  e
!e"Z#ej$%e"d4Z&e'Z(d5d6 Z)G d7d8 d8Z*ej+G d9d: d:Z,G d;d< d<e,Z-e.d=d> e/e	0 D Z1e.e2 e3d?< e4 Z5e4e2 e3d@< ej6j7j8Z9ej6j7j:Z;G dAdB dBZ<dCdD Z=dEdF Z>dtedGdHdIZ?dJdK Z@dudLdMZAdvdNdOZBdwdPdQZCdRdS ZDdTdU ZEdVdW ZFdaGee4eH  e3dX< daIee2 e3dY< e4eH dGdZd[ZJeHeKd\d]d^ZLd_eHejMead`dadbZNdaOee4eH  e3dc< daPee2 e3dd< e4eH dGdedfZQeHeKd\dgdhZRdxe?dGdjdkZSddlee| eKdmdndoZTG dpdq dqZUeUV eU_WG drds dsZXdS )yaq  
This module contains classes and utilities for building variable trackers in Dynamo.
Variable trackers are used to convert Python values into symbolic representations
that can be traced and transformed during graph capture.

The key classes are:

- VariableBuilder: Handles source-tracked objects that need guards and proper
  reconstruction in the output graph. Used for inputs, module attributes, etc.

- SourcelessBuilder: Handles ephemeral objects created during tracing that don't
  need source tracking or guards. Used for temporary lists, intermediate values, etc.

Variable trackers enable Dynamo to track the flow of values through the program,
maintain guards for dynamic properties, and reconstruct values in the output graph.
The builders in this module handle converting Python values into appropriate
VariableTracker instances based on their type and usage context.
    N)MutableMapping)AnyCallable
NamedTupleOptionalTYPE_CHECKINGUnion)SymInt)get_metrics_contextis_int_specialization_caseis_torch_symset_feature_use)TracingContext)call_torchbind)HigherOrderOperator)
FakeTensoris_fakemaybe_get_fake_mode)is_sparse_any	safe_grad)justknobs_check)BackwardState)normalize_source_name)	_constrain_range_for_size_nested_int_aware_sort
DimDynamicRelaxedUnspecConstraintStatefulSymbolicContextSubclassSymbolicContextSymbolicContextSymIntSymbolicContextTrackedFake)immutable_dictimmutable_list)ExpandedWeight)is_traceable_wrapper_subclass"is_traceable_wrapper_subclass_type)ValueRanges)TensorWeakRef   )configgraph_break_hintsmutation_guardreplay_recordtrace_rules) get_registered_device_interfaces)InternalTorchDynamoErrorraise_observed_exceptionunimplemented_v2)GuardBuilderinstall_guardmake_dupe_guard)auto_dynamic
auto_unsetFrameStateSizeEntryInferStrideprocess_automatic_dynamic)SideEffects)AttrProxySource
AttrSourceCallMethodItemSourceChainedSourceConstDictKeySourceConvertIntSourceDictGetItemSourceDictSubclassGetItemSourceFloatTensorSourceGetItemSource
GradSourceis_constant_sourceis_from_global_sourceis_from_nonlocal_sourceis_from_optimizer_source&is_from_unspecialized_nn_module_sourceListGetItemSourceLocalSourceNumpyTensorSourceOptimizerSourceRandomValueSourceSourceSubclassAttrListSourceTupleIteratorGetItemSource"UnspecializedBuiltinNNModuleSourceUnspecializedNNModuleSource)_extract_tensor_dictbuild_checkpoint_variablebuild_invoke_subgraph_variableclone_inputcommon_constant_types	dict_keysget_fake_valueget_items_from_dictget_locals_to_stealget_static_address_typeis_frozen_dataclassis_function_or_wrapperis_invoke_subgraphis_lru_cache_wrapped_functionis_namedtupleis_parameter_freezing	is_typingis_utils_checkpointis_wrapper_or_member_descriptoristypenamedtuple_fieldsodict_valuesproxy_args_kwargsrange_iteratorset_example_valuetensor_always_has_static_shapetuple_iteratortuple_iterator_getitemtuple_iterator_len unwrap_with_attr_name_if_wrapperwrap_fake_exception   )AttributeMutationNewtypestrValueMutationExistingValueMutationNewVariableTrackerVariableTrackerMeta)ConstantVariableEnumVariable)AutocastModeVariableDynamoConfigPatchVariableEventVariableNullContextVariablePreserveVersionContextVariableStreamContextVariableStreamVariable)ConstDictVariableDefaultDictVariableDictKeySetVariableFrozensetVariableMappingProxyVariableSetVariable)DeviceMeshVariablePlacementClassVariablePlacementVariableProcessGroupVariableWorldMetaClassVariable)BuiltinMethodVariableCollectionsNamedTupleFunction!CollectiveFunctionRewriteVariable'CreateTMADescriptorExperimentalVariable!CreateTMADescriptorStableVariableFunctoolsPartialVariableFunctoolsWrapsVariableSysFunctionVariableTracebackVariableTritonKernelVariableUserFunctionVariableUserMethodVariableWrapperUserFunctionVariable) TorchHigherOrderOperatorVariable)ItertoolsVariable)LazyVariableTracker)	BaseListVariableListIteratorVariableListVariableNamedTupleVariableRangeVariableSizeVariableSliceVariableTupleIteratorVariableTupleVariable)AutogradEngineVariableAutogradFunctionContextVariableAutogradFunctionVariableComptimeVariableDebuggingVariableDelayGraphBreakVariableGetAttrVariableGetSetDescriptorVariableLambdaVariableLoggingLoggerVariableMethodWrapperVariableNumpyDTypeVariableNumpyTypeInfoVariableNumpyVariablePythonModuleVariableRandomClassVariableRandomVariableRegexPatternVariableSavedTensorBoxTorchVersionVariableTypingVariableWeakRefVariable)FSDPManagedNNModuleVariable$UnspecializedBuiltinNNModuleVariableUnspecializedNNModuleVariable)OptimizerVariable)TorchScriptObjectVariableSDPAParamsVariable)NumpyNdarrayVariable$supported_const_comparison_op_valuesSymNodeVariableTensorSubclassVariableTensorVariableUnspecializedPythonVariable)DispatchKeySetVariableFuncTorchInterpreterVariableTorchCtxManagerClassVariableTorchInGraphFunctionVariable)TensorWithTFOverrideVariable#torch_function_mode_stack_state_mgrTorchFunctionModeVariable)FrozenDataClassVariableIntWrapperVariableKeyedJaggedTensorVariableMutableMappingVariableSourcelessGraphModuleVariableUserDefinedClassVariableUserDefinedDictVariable!UserDefinedExceptionClassVariableUserDefinedListVariableUserDefinedObjectVariableUserDefinedTupleVariable)	PyCodegen)InstructionTranslatorZcudagraph_static_inputsc                 C   sB   t  & t dd t| dW  d    S 1 s40    Y  d S )NignorezThe .grad attribute of a TensorZgrad)warningscatch_warningsfilterwarningshasattr)t r   M/var/www/auris/lib/python3.9/site-packages/torch/_dynamo/variables/builder.pysafe_has_grad/  s    
r   c                   @   s   e Zd ZdS )_missingN__name__
__module____qualname__r   r   r   r   r   5  s   r   c                   @   s   e Zd ZU eed< eeejf ed< e	ed< e
ejjj ed< dZe	ed< dZe
ej ed< ed	d
 Zdd ZddddZdd Zdd ZdS )GraphArgsource_examplepass_arg_as_tensorfake_tensorT	is_tensorNexample_strong_refc                 C   s.   t | jtr$|  }|d us J |S | jS d S N)
isinstancer   r(   )selfrr   r   r   example]  s
    zGraphArg.examplec                 C   s,   t | jtjr(t| j| _t| js(J d S r   )r   r   torchTensorr(   r   r   r   r   r   r   __post_init__f  s    zGraphArg.__post_init__r   codegenc                 C   s   || j  d S r   r   r   r   r   r   r   reconstructk  s    zGraphArg.reconstructc                 C   s   d | _ d | _d S r   )r   r   r   r   r   r   erasen  s    zGraphArg.erasec                 C   s   | j  |j  kS r   )r   name)r   otherr   r   r   __eq__r  s    zGraphArg.__eq__)r   r   r   rQ   __annotations__r   r(   r   r	   boolr   _subclassesr   r   r   r   r   propertyr   r   r  r  r  r   r   r   r   r   9  s   

r   c                       s0   e Zd Zdd fddZddddZ  ZS )	BackwardStateGraphArgNreturnc                    s   t  jd t dd dd d S )NF)r   r   r   r   r   )super__init__r   r   	__class__r   r   r  w  s    zBackwardStateGraphArg.__init__r   r   c                    sH    j jjsJ   fdd  dd      j jj d S )Nc                      s     tjdS )Nr   )Zload_import_fromr   r   r   r   r   r   <lambda>      z3BackwardStateGraphArg.reconstruct.<locals>.<lambda>r   F)txoutputZbackward_state_varZadd_push_nullcall_functionZdup_topstorer  r   r   r   r    s    
z!BackwardStateGraphArg.reconstruct)r   r   r   r  r  __classcell__r   r   r  r   r  v  s   	r  c                 c   s.   | ]&\}}| d st|rt|V  qdS )_N)
startswithinspectisclassid).0r  memberr   r   r   	<genexpr>  s   r   ITERTOOLS_TYPE_IDSITERTOOLS_POLYFILLED_TYPE_IDSc                       s  e Zd ZdZedd fddZdd Zdd	 Zd
d Zdd Z	e
dd Ze
ejdd ZejdddZejdddZdd Zdd Zdd Ze
ejeeed egef f dddZd d! Zedd"d#Ze e!e"e#e$f dd$d%Z%e&dd&d'Z'e(dd(d)Z)e e*e+f dd*d+Z,e-j.e/d,d-d.Z0e-j1j2dd/d0Z3d1d2 Z4e-j.dd3d4Z5e-j.dd5d6Z6d7d8 Z7d@e8e9 e8e: d9d:d;Z;d<d= Z<d>d? Z=  Z>S )AVariableBuilderz3Wrap a python value in a VariableTracker() instanceN)r   r  c                    sH   |d usJ dt  d us$J dt   || _|| _| | _d S )NzRConsider SourcelessBuilder for ephemeral objects, usually objects created locally.zExpected active TracingContext)r   Ztry_getr  r  r  r   r  )r   r  r   r  r   r   r    s    

zVariableBuilder.__init__c                 C   s   || j jjv r<| j jj| }t| j|j}|r8| | |S | j jj|| j}|rX|S | |}|jd u rt| j|_| 	|r|| j jjvrt
|s| j jj||}| j jj|| j| |S r   )r  r  side_effectsr5   r   install_guardsZvariable_tracker_cachelookup_wrap_can_lift_attrs_to_inputsrh   track_object_existingadd)r   valueZside_effect_resultZ	dup_guardZ	cached_vtvtr   r   r   __call__  s*    


zVariableBuilder.__call__c                 C   s   t |tttthv S r   )typer   r   r   r   )r   r,  r   r   r   r(    s    z)VariableBuilder._can_lift_attrs_to_inputsc                 C   s   | j S r   r   r   r   r   r   
get_source  s    zVariableBuilder.get_sourcec                    sF   |    z fdd|D }W n ty2   Y d S 0 t|ddi i S )Nc                    s   g | ]}  |qS r   )
make_guard)r  guardr   r   r   
<listcomp>  r  z2VariableBuilder.install_guards.<locals>.<listcomp>skipru   )r/  NotImplementedErrorr4   )r   guardstmpr   r   r   r%    s    zVariableBuilder.install_guardsc                 C   s   |  tjS r   )_type_dispatch_implr*   trace_numpy)clsr   r   r   _type_dispatch  s    zVariableBuilder._type_dispatchc                 C   s   t jt jjt jjt jjjf| jft	t
ttjt jf| jft| jft| jfttf| jft	t| jftj| jftj| jft jj j!| j"ft j#j$| j%ft&j'| j(fg}|rt)r|*t)j+| j,f i }|D ]6\}}t-|t	r|n|fD ]}||vsJ |||< qq|S r   ).r   r   nn	Parameterr	  r   functional_tensorFunctionalTensorwrap_tensortuplelistrk   collectionsdequeSizewrap_listlikerp   wrap_tuple_iteratorrm   wrap_range_iteratorslicerangewrap_slice_rangerZ   wrap_literalrePatternwrap_regex_patternweakrefReferenceTypewrap_weakrefutilshooksZRemovableHandlewrap_removable_handleZjitZScriptFunctionwrap_jit_functiontypesMappingProxyTypewrap_mapping_proxynpappendndarraywrap_numpy_ndarrayr   )r9  r8  entriesresulttsfnr   r   r   r   r7    s8    



z#VariableBuilder._type_dispatch_implr+  c                 C   s   |  tj t|S r   )r%  r3   ID_MATCHr   r   r+  r   r   r   rN    s    z"VariableBuilder.wrap_regex_patternc                 C   s    |  tj tj| j|| jdS Nr   )r%  r3   
TYPE_MATCHr   buildr  r   rc  r   r   r   rQ    s    zVariableBuilder.wrap_weakrefc                 C   s   t dddg d d S )Nz3Attempted to represent unregistered RemovableHandle zDynamo attempted to build a representation of a torch.utils.hooks.RemovableHandle, which is not supported. This happens because the RemovableHandle was created in another frame.gb_typecontextexplanationhints)r2   rc  r   r   r   rT    s    z%VariableBuilder.wrap_removable_handlec                 C   s   |  tj t|d| jdS )N_torchdynamo_inliner   )r%  r3   re  r   r   rc  r   r   r   rU    s    z!VariableBuilder.wrap_jit_functionc                    s    tj  tj tdd | D }|sXtdddd | D  ddgd	 fd
d t fdd| D }t	|d d}t
|jd}jjj||S )Nc                 s   s   | ]}t |V  qd S r   r|   
is_literalr  kr   r   r   r   (  r  z5VariableBuilder.wrap_mapping_proxy.<locals>.<genexpr>znon-const keys in mappingproxynon-const keys: c                 S   s   g | ]}t |s|qS r   rn  rp  r   r   r   r2  -  r  z6VariableBuilder.wrap_mapping_proxy.<locals>.<listcomp>z1Dynamo expects mappingproxy keys to be constants.zFEnsure your mappingproxy keys are constants (e.g. int, float, strings)rh  c                    s0   t | }| }t  |}t||}||fS r   )r|   createrE   r/  r   )rq  vkey
source_keysource_value	res_valuer   r   r   build_key_value4  s
    
z;VariableBuilder.wrap_mapping_proxy.<locals>.build_key_valuec                 3   s   | ]\}} ||V  qd S r   r   r  rq  rt  ry  r   r   r   =  r  r   )r%  r3   re  ZMAPPING_KEYS_CHECKallkeysr2   dictitemsr   r   r   r  r  r$  track_mutable)r   r+  	all_constr  dict_vtr^  r   )ry  r   r   rX  #  s     		z"VariableBuilder.wrap_mapping_proxyr  c                 C   s~   ddl m } |dd ftjdd ftjdd fg}i }|D ]>\}}t|ttfrT|n|fD ]}||vsjJ ||t|< qZq:|S )Nr)   )comptimec                 S   s   t  S r   )r   rc  r   r   r   r  L  r  z.VariableBuilder._id_dispatch.<locals>.<lambda>c                 S   s   t tfd| ji| tjS )Nr   )r   _dataclasses_fields_lambdar   r%  r3   FUNCTION_MATCHrc  r   r   r   r  O  s   
c                 S   s   t  S r   )r   rc  r   r   r   r  U  r  )	r  dataclassesfieldsr   __version__r   r@  rA  r  )r9  r  r]  r^  r_  r`  r   r   r   r   _id_dispatchD  s    
zVariableBuilder._id_dispatchc           4   	      s~  ddl m}m}m} ddlm} | r@ddlm} ddlm	} nG dd d}G d	d
 d
}dd }dd }	G dd d}
| rddl
m}m}	 | rddlm}
  t}|d ur|S  t}|d ur|S ttjrZttjjjtjjjtfvrZttjvrZtjtjju rFS t rZS t!r"t#j$ fddt%tD }t&|tj'd}|S t(t)t*j+t*j,fr"t#j- t.dd / D   sj0j1j23j'  fddt)fddt4t5D }t(t*j+r`t6j'd}t7|tt8j0|j9j'd}nt:|tj'd}j0j1j;<|S ttjj=r>S t?@rAS ttjBjCrtDj'd}j0j1j;E| |S t(tFr6t.dd D r6fd dD }"t#jG tH|j'dS ttIjJtjKtjLjMjNfrl"t#jG tOj'd!S tPQr"t#jR tPj'dS ttSjTr"t#jU tVj'dS tWrtXj'dS tYrtZj'dS tt[j\rt6] d"}t8j0|j^}g }t6] d#}t4j_D ]&\}}|`t8j0ta||| q0i }t6] d$}jbc D ]L\}}t?@|stdd%d&| d'g tejfd( t8j0tg|||||< qtth] it#j-|it#jj|it#j$ tk|||S tlr"t#jG tmj'dS tnd urBttnjorBptnqS trsrtnsXJ "ttrlt#jUnt#j- tuj'dS trvr"t#jG twj'dS trxrttnjyr"t#j- t6j'd)}th|it#jG n"t#jG tzj'dS t{|r2"t#jU t{j}j0j'dS t(tj~jjr^"t#jU tj'dS ttj~jjr,d }z
j}W n ty   Y n0 g }j'it#j-g}t|trt6j'd*} |`| it#j$ t4|D ]&\}}|`t8j0ta| || qth|  j0j1j;Etj't|d+S ttjrt(td,d tj~jjrtd-d.d/krtjd/d kr"t#jU ttjt6j'd,d0dd/S ttjLjrΈ"t#jG tj'dS tjjjju r"t#jU tfd1d2S t|rtjS ttrXtrd urXtrrBd3j0j1_trjj'dS tnrvttnjrvS ttrtjju rtdd4d.d5g d( "t#j-t#j tjj'dS ttjj	r"t#jG t6j'd6}!t8j0|!j}"t}j0|"S ttj	rh"t#jG j0j1d7td8jjjd9}#t|#j t|#jj'dS ttjLj	r"t#j- t}j0j'S ttjMjj	r"t#jG tS ttj
r "t#jG tjj j0j1d7tjjjtfi }$t|$j t|$j'dS t(tj
rZtd:d d u 
rZ"t#j- tj'dS t
r"t#j- tj'd}j0j1j;E|S ttjj
rȈ"t#jG tj'_'tÈj'dS ttjă
r"t#jš tƈS tǠȈrtǈj'dS tɠʈr*"t#jG tɈj'dS tˠ̈rP"t#j͡ tˈj'dS tΠψrv"t#jG tΈj'dS tРшr"t#j͡ tЈj'dS ttv rttvr҈"t#jU tԈj'dS tՈr ttjփrj'nt׈j'}%jء r(j0j1jjtۈjj܃|%tjd;}&n4ttj߃rFj0j1j٠ }&ntdd<d.d=g tejd( j0j1jjtd>d?jt|&|&|%d}'t|%|&d@d d@|&dA|'jjdB< |&jj}(t|(tjsJ |( dCj0j1j`t|&|%d  ttjփr|&n|&dDk})t|'|)S t||fr."t#jG td d j'dS |u rBtdDdES |	u rVtddES |
ju rht S ttjjjr"t#jG tjjjjgj'dFS trʈ"t#jU tj'dS tdGd@r"t#j- tdHj'dS tr"t#j- tdIj'dS tju r6t j'dS tju sftjdJkrvtju rvtj'dS trtdKd@r"t#j- tdKj'dS t[ju r؈"t#jG tj'dS t*j	u r"t#jG t
j'dS ttjr>tr>"t#jG tj'dS trt\}*|*d urpt6j'|*_'trjj'dS tju r"t#jG tj'dS t(tjrtr"t#j- tj'd}j0j1j;<| |S ttjtjfrR"t#jU tj'd}j0j1j;E| |S ttjrtjtjj=tjjjfrt8j0t6j'd,dj}+|+rt|+tsJ dL"t#jU tj|+j'dS ttjrtS ttj rt!S t"ttrFt"t#rF"t#jG t$S t"ttr&tjj%j&tjj'tjj(fv rtrjj'dS tj~j)u r"t#jU t*+j0S t"tjr
tjur
jtjju r
t,s
t-j'dS "t#jG t.j'dS t/trddMl'm0},m1}- tj2j34rj0j1jjtd>d?jtj'd}.tj'd@d d@|.jjdB< t/j}|.j'dS t5dNsֈ6S t78t8j0|-j'9 :  t78t8j0|,j';  tj2j3<j0j1j=}/j0j1jjtd>d?jt|/j'd}.tj'd@d d@|/|.jjdB< t/j}|.|/j'dS tt)t*j,frrtj>t)j>u rr"t#j- "t#j$ j0j1j23j' fdOdt)fdPdt4t5D }t:|tt*j,r4t*j,nt)t? j'dQ}0d3|0_@tA|0j'dR}j0j1j;E|S ttr"t#j- "t#j$ fdSdtBtCD }tD|j't? dT}1tE|1j'dU}j0j1j;E|S ttFrv"t#j- "t#j$ fdVdtBtFCD }tG|j't? dT}2tH|2j'dW}j0j1j;E|S t"ttIr"t#j- tJj'dS tKr"t#j- tLj}j0j'd}j0j1j;E|S ttMrrt.dXd D rLfdYdD }th] it#j$] it#j͡ tN|j'dS tddZd[d\d D  d]d^gd( ntOrndd_lPmQ}3 jRd u sjRj|3jSkrTjUS jRj|3jkrtVWd`j' tjޡ jTjUtjސtXtYd@dadbdcS jRj|3jZkrZtVWd`j' tjޡ jTjUtjddS tdejR n6S d S )fNr   )
has_triton has_triton_experimental_host_tma%has_triton_tensor_descriptor_host_tmar)   )DynamoConfigPatchProxy)	Autotuner)JITFunctionc                   @   s   e Zd ZdS )z*VariableBuilder._wrap.<locals>.JITFunctionNr   r   r   r   r   r  o  s   r  c                   @   s   e Zd ZdS )z(VariableBuilder._wrap.<locals>.AutotunerNr   r   r   r   r   r  r  s   r  c                   S   s   d S r   r   r   r   r   r   create_1d_tma_descriptorv  s    z7VariableBuilder._wrap.<locals>.create_1d_tma_descriptorc                   S   s   d S r   r   r   r   r   r   create_2d_tma_descriptory  s    z7VariableBuilder._wrap.<locals>.create_2d_tma_descriptorc                   @   s   e Zd Zedd ZdS )z/VariableBuilder._wrap.<locals>.TensorDescriptorc                   S   s   d S r   r   r   r   r   r   from_tensor}  s    z;VariableBuilder._wrap.<locals>.TensorDescriptor.from_tensorN)r   r   r   staticmethodr  r   r   r   r   TensorDescriptor|  s   r  )r  r  )r  c                    s(   g | ] }t jt|t j|d qS r   )r   rs  getattrr=   r   r  r  rc  r   r   r2    s
   
z)VariableBuilder._wrap.<locals>.<listcomp>)	tuple_clsr   c                 s   s   | ]}t |V  qd S r   rn  rp  r   r   r   r     r  z(VariableBuilder._wrap.<locals>.<genexpr>c                    sP     } rt|}|}nt|| }t||}t||}t||}||fS r   )r/  r|   rs  r@   r   rB   )irq  rt  baseru  rv  rw  rx  )r  r   r   r   ry    s    


z.VariableBuilder._wrap.<locals>.build_key_valuec                 3   s"   | ]\}\}} |||V  qd S r   r   r  r  rq  rt  r{  r   r   r     s   
default_factory)r  r   )user_clsr   r   c                 s   s4   | ],}t |tju r|jd kp*|tjjjv V  qdS )r   N)r.  rV  BuiltinMethodTyper   r   rR  _pytreeZBUILTIN_TYPESr  xr   r   r   r     s   c                    s   g | ]}t  j|qS r   SourcelessBuilderrs  r  r  rt  r   r   r   r2    r  r+  r   funcargskeywordsz,functools.partial() with non-literal keywordznon-literal keyword: z3functools.partial() expects literal/string keywordsrh  dtypesaved_tensors)r   r  __self__r   rg  apply)r  c                      s(   t tjjjj j jjj	
 fi S r   )r   r   _dynamoexternal_utilsFakeCompiledAutogradEngineZexec_final_callbacksr  r  r  r$  Zget_ca_final_callbacks_varr   r   r   r   r    s   
z'VariableBuilder._wrap.<locals>.<lambda>Tz9Attempted to wrap torch._higher_order_ops.invoke_subgraphzJDirectly using invoke_subgraph is not supported. Use nested_compile_regionstreamr  r   )	stream_iddevice_indexdevice_typeenter_result)dynamic_dimz!Attempted to wrap unbacked SymIntz+Unbacked SymInt input is not supported yet.[^a-zA-Z0-9]+r  F)r   r   grapharg is not a basic Symbol.ru   )Zrank)Ztarget_valuesr   Z__script_if_tracing_wrapperZ__original_fn__wrapped__)      rm  z"Failed to produce a valid self obj)FlattenScriptObjectSourceScriptObjectQualifiedNameSource__obj_flatten__c                    s<      }t|| }t||}t||}t||}||fS r   )r/  r@   r   rs  rC   )r  rq  rt  r  rv  ru  rw  rx  r   r   r   ry  X  s    

c                 3   s"   | ]\}\}} |||V  qd S r   r   r  r{  r   r   r   f  s   
)r  mutation_typer   )r  r   c                    s,   g | ]$}t jt|t  |d qS r  )r   rs  r@  __getitem__rE   r/  r  r  rc  r   r   r2    s
   
)r   r  )tuple_vtr   c                    s,   g | ]$}t jt|t  |d qS r  )r   rs  rA  r  rL   r/  r  rc  r   r   r2    s
   
)list_vtr   c                 s   s   | ]}t |V  qd S r   rn  rp  r   r   r   r     r  c                    s   g | ]}t  j|qS r   r  r  r   r   r   r2    r  znon-const keys in dict_keysrr  c                 S   s   g | ]}t |s|qS r   rn  rp  r   r   r   r2    r  z.Dynamo expects dict_keys keys to be constants.zCEnsure your dict_keys keys are constants (e.g. int, float, strings))_DimHintTypez%s marked %s via IntWrapper	warn_only)
constraintdynamismrj  r  zUndefined dynamism ([  Ztorch.utils._tritonr  r  r  
decoratorsr  Ztriton.runtime.autotunerr  Ztriton.runtime.jitr  Z$triton.tools.experimental_descriptorr  r  Ztriton.tools.tensor_descriptorr  r:  getr.  r  r  r   r   r   r;  Z	parameterZUninitializedBufferZUninitializedParameterr$   r*   Znontraceable_tensor_subclassesZ__torch_dispatch__r?  r%   rd   r%  r3   SEQUENCE_LENGTHrj   r   r   ri   r~  rB  defaultdictOrderedDictre  r|  r}  r  r  Zguard_on_key_orderr*  	enumerater]   r=   r   r#  r  r   r$  r  Modulewrap_moduler|   ro  rK  Z	overridesZTorchFunctionModer   r)  	frozensetrb  r   enumEnumDispatchKey_C
_functorchTransformTyper}   r   Zis_reorderable_logging_functionZBUILTIN_MATCHloggingLoggerr  r   rg   rW   rb   rX   	functoolspartialr/  r  r  rZ  rE   r  r  r2   r+   
USER_ERRORrB   r4   r0  ZDICT_KEYS_MATCHr   rf   r   rY  Zgenericr\  Zasarrayr.   Zis_numpycallabler   Zis_numpy_dtyper   Zis_numpy_type_infoZiinfor   r   Zcan_rewriters  ZautogradfunctionZFunctionMetar   ZFunctionCtxr  RuntimeErrorr@  r   r   rV  
MethodTyper  r  r   Z_ImperativeEnginer   r  r  r  Z_exec_final_callbacks_stubr   r   changeslookup_callableis_callable_allowed!has_user_defined_allowed_in_graphZcreate_with_sourcenumberwrap_unspecialized_primitiver   _higher_order_opsZinvoke_subgraphZ
NAME_MATCHr   makecudaZStreamContextr  r   Streamcreate_proxyr  r  r  rn   noder   deviceZ_SDPAParamsr   pyfunctorchFuncTorchInterpreterr   EventrR  Zstore_user_object_weakrefZget_user_object_from_idr   
contextlibnullcontextr  getattr_staticr   r   Zis_matching_objectZoptimZ	OptimizerrO   r   DispatchKeySetZDISPATCH_KEY_SET_MATCHr   r   Zis_group_member_typer   Zis_process_groupr   is_device_meshEQUALS_MATCHr   Zis_placement_typer   is_placementr!  r"  r   r   r	   rA   Zhas_hint	shape_env$create_unspecified_symint_and_symbolinthintr   DYNAMICSymBoolcreate_unbacked_symintSUPPORTABLEroot_tracercreate_graph_inputrL  subr  r   metaexprsympySymboltracked_fakesr!   r   r   r   r  r   ampZautocast_modeZautocastr~   Z
fast_dtypeZ_enabledZ_cache_enabledr   Zis_matching_clsr   rc   	tracebackclear_framesr   sysexc_infoversion_info	exceptionr   ra   wrapsr   
namedtupler   r  r   Zis_supported_builtin_methodrs   r&  randomRandomr   r   Zis_supported_random_obj
ModuleTyper-   ZDummyModuler   r  TreeSpecrz   r   __func__GetSetDescriptorTyper   MethodWrapperTyper   
issubclassBaseExceptionr   rS  ZBackwardHookr<  BufferZ _unsafe_preserve_version_counterr   constructorr&   r   r   r   r  r  Z_libraryZfake_class_registryZtracing_with_realr   wrap_user_definedr   realize_all_typeZqualified_namer  Zmaybe_to_fake_obj	fake_mode__new__rx   Zshould_reconstruct_allr   rI  __len__r   r   rA  r   r   r   r   r`   r   r[   r   r   torch.export.dynamic_shapesr  r  STATICwrap_symintvallogdebugr    r   ZAUTO)4r   r+  r  r  r  r  r  r  r  r  r  Ztype_dispatchZid_dispatchr  r^  Zfactory_sourcevarr  Zfunc_srcZfunc_objr  Zargs_sourcer  argr  Zkeywords_sourcerq  rt  Z	dt_sourceZactual_saved_tensorsr  r5  Zsaved_tensors_sourceZstream_sourceZ
stream_varZstream_proxyZevent_proxyr   Z
new_symintZsym_node_proxysym_exprZtracing_symint	attr_nameZself_objr  r  proxyZfake_script_objr  r  r  r  r   )r  ry  r   r+  r   r'  `  s   









	












	
	

	













	




	



"








"
zVariableBuilder._wrapc                 C   s>   |  tj t|| jd}tt|s,|S | jj	j
||S rd  )r%  r3   re  r   r   r;   "cls_supports_mutation_side_effectsr.  r  r  r$  r)  )r   r+  r^  r   r   r   r    s
    z!VariableBuilder.wrap_user_definedc                    s$  |D ]}||u rt ddddgd qtjrPt|tju rP tj t	j
|dS  tj t|trtdd |D r j  r tj td	d
 |D S  fdd
t|D } jjjd}t jtr jjt|v r j}t|tsJ  jjjjt dd j!t|||d}d|j"j#d< t$t% j||d |d}g }t|j&D ]d\}	}
t'||	dd}|
 jjj(|< t)||	 |
j*j"j#d< t+j,tj-t.||	 d}|/|0| qVt1|ddi t2||dd dd}||j"j#d< t34|| jd}t|tt5j6fr  jjj78||S |S )Nz-list elements are pointing to the list itselfrg  z=Dynamo does not support lists whose items reference to itselfz!Avoid using self referential listrh  ra  c                 s   s   | ]}t |V  qd S r   rn  r  itemr   r   r   r     r  z0VariableBuilder.wrap_listlike.<locals>.<genexpr>c                 S   s   g | ]}t |qS r   r|   rs  r0  r   r   r   r2     r  z1VariableBuilder.wrap_listlike.<locals>.<listcomp>c                    s(   g | ] \}}t j|t  |d qS r  )r   rs  rE   r/  )r  r  r1  r   r   r   r2    s
   r   r  r  r   TZ	steal_arg)
target_clsr  r.  example_valuesubclass_typer   Fr  indexZindex_is_slicetensor_dictr3  ru   )r   r   r   r  )9r2   r*   specialize_intr.  r   rD  r%  r3   CONSTANT_MATCHr|   rs  r  ri   r@  r|  r   guard_sourceis_unspecialized_nn_moduler   r  r  r  Zlocal_scoper  r   rM   Z
local_namer^   rA  r  r  rL  r  r  r  r  wrap_fx_proxy_clsr   r  rE   input_source_to_varrV   r.  r  r  TENSOR_MATCHr(   rZ  r0  r4   r   r   Zcls_for_instancerB  rC  r$  r  )r   r+  r1  r  Zmaybe_gmr   Ztensor_list_proxyZlist_variabler5  r  tensor_variableZsource_ir1  r  r^  r   r   r   rE    s    

	zVariableBuilder.wrap_listlikec                    sH     tj  fddttD }t| jd} jjj	
|S )Nc                    s,   g | ]$}t  jt  |t|qS r   )r#  r  rS   r/  rq   r  rc  r   r   r2  I  s   z7VariableBuilder.wrap_tuple_iterator.<locals>.<listcomp>r   )r%  r3   ZTUPLE_ITERATOR_LENrI  rr   r   r   r  r  r$  r  )r   r+  r  r^  r   rc  r   rF  G  s    
z#VariableBuilder.wrap_tuple_iteratorc                 C   s@   |  tj dd t|D }t|| jd}| jjj	
||S )Nc                 S   s   g | ]}t |qS r   r2  r  r   r   r   r2  V  r  z7VariableBuilder.wrap_range_iterator.<locals>.<listcomp>r   )r%  r3   ZRANGE_ITERATOR_MATCHcopydeepcopyr   r   r  r  r$  r  )r   r+  r  r^  r   r   r   rG  R  s    z#VariableBuilder.wrap_range_iteratorc                    sJ    fdddD }  tj ttr8t| jdS t| jdS d S )Nc                    s,   g | ]$}t  jt  |t|qS r   )r#  r  r=   r/  r  rp  rc  r   r   r2  [  s   z4VariableBuilder.wrap_slice_range.<locals>.<listcomp>)startstopstepr   )r%  r3   re  r   rH  r   r   r   )r   r+  r  r   rc  r   rJ  Z  s    
z VariableBuilder.wrap_slice_range)r+  r1  c                 C   sd   ddl m} td| j t| |||d || jjj	v r`| jjj	| }|j
|jjjd d< d S )Nr)   )mark_static_addressz Marking static input %s, id: %s)r1  r8  _dynamo_static_input_type)r  rF  static_inputs_logr)  r   r  r  r  r  r$  rH  r.  r  r  )r   r+  r1  rF  r*  r   r   r   mark_static_inputg  s    z!VariableBuilder.mark_static_inputc              
   C   s  ddl m} t|jdkrDtdt|dt| dg tjdd t||rt	
|jd	d
rt	
|jdd }t| jd| ddS | tj t| jd| _| |jS t|tjjtjjtjjfrtjstdt|dg tjd t|dd
rTt|dd
stdddg d | tj t||   d}t!"t#|sB|S | j$j%j&'||S t()|| j$j*rt|tj+j,j-j.r|/ }t0| j| _tj1jj2rt3 }t4|j5r"|j5j6t7u r"z&|5 D ]\}}| j8||d qW n@ t9y  } z&t:t#|| j$t;|j<d W Y d }~n
d }~0 0 t4|j=r|j=j6t>u rz&|= D ]\}}	| j8|	|d qFW n@ t9y } z&t:t#|| j$t;|j<d W Y d }~n
d }~0 0 |r|| j$j%j?| j@< |jABdr|jABdrt|jCdd
r0| j}
tj2r| j$j%j*rtjDrtE| j}
tF||
d}tG|
Htj nH| j}
tj2r\| j$j%j*rRtjDr\tI| j}
tJ||
d}tG|
Htj t!"t#|s|S | j$j%j&'||S tK|jCtjjLjMjNr| tj tJ||   dS | j$j%jO|| j@|   dS d S )Nr)   )OptimizedModuler   zUninitialized nn.Modulez6Attempted to trace an uninitialized nn.Module of type .z?Ensure your nn.Module instance has called `super().__init__()`.rh  Z_torchdynamo_disableFZ_torchdynamo_disable_msgzHOptimized `nn.Module` is wrapped with `torch.compiler.disable` (reason: ))r   msg	_orig_modz#Attempted to wrap RNN, GRU, or LSTMz*Dynamo does not support RNN, GRU, or LSTM.Z_is_fsdp_managed_moduleZ_fsdp_use_orig_paramszFSDP with use_orig_params=Falserg  z3Dynamo only supports FSDP with use_orig_params=Truer   rG  )r  )ztorch.nn.modulesz	torch.ao.ztorch.nn.modules.containerZ_dynamo_marked_static)PZ
eval_framerK  len__dict__r2   rw   r+   r  ri   r  r  Zforwardr   r   r%  r3   re  r=   r  rO  r   r   r;  ZRNNZGRUZLSTMr*   Z	allow_rnnstrr  r  r   r/  r;   r/  r.  r  r  r$  r)  r,   Zis_dynamic_nn_moduleexportfxZexperimentalZproxy_tensorZ
_AttrProxyget_baser<   r  inline_inbuilt_nn_modulesre   r  named_parametersr  !og_module_named_parameters_fn_ptrrJ  	TypeErrorr1   rA  r  named_buffersog_module_named_buffers_fn_ptrZ
nn_modulesr  r   r  r  install_free_tensorsrT   r   r4   r0  rU   r   r  paralleldistributedZDistributedDataParallelregister_attr_or_module)r   r+  rK  rN  r^  Zfreezingr  pebZ
new_sourcer   r   r   r  w  s    	

0
0



zVariableBuilder.wrap_modulec                 C   s4  t |tu rt| j r<td| j  | j|tj	dS t
| j rltd| j  | j|tjdS tjst|| jrd }| j  s| j  rd}| tjtj|d tj|| jdS | |S tjst |tu r| |S | tj tj|| jd}t|ttfr,| j j!j"#||S |S d S )N&%s marked dynamic via source whitelistr  '%s marked unbacked via source whitelista  torch.compile considers integer attributes of the nn.Module to be static. If you are observing recompilation, you might want to make this integer dynamic using torch._dynamo.config.allow_unspec_int_on_nn_module = True, or convert this integer into a tensor.)recompile_hintr  )$r.  r  is_dynamic_sourcer   r  r(  r)  r&  r   r  is_unbacked_sourceSIZE_LIKE_UNBACKEDr*   r9  r   r;  Z"is_unspecialized_builtin_nn_moduler<  r%  r  r  r3   r  r|   rs  specialize_floatfloatwrap_symfloatr:  r   rA  setr  r  r$  r  )r   r+  re  r^  r   r   r   rK    s<    


zVariableBuilder.wrap_literalc                 C   s&   t |r"t|| jju r"tddd S )Nz*Cannot wrap a Tensor that has already beenz"wrapped by this instance of Dynamo)r   r   r  r!  r0   rc  r   r   r    assert_not_wrapped_by_this_graphC  s
    z0VariableBuilder.assert_not_wrapped_by_this_graphc                 C   sp  |   }|| jjjvsJ t|d u}tjrb|sbt|tj	j
sN|rb|  rb| j|t d d}tjo~t|p~t|p~t|}|otj pt ptjjj}|s|  s|r|  s| | | jjj|| j|dS t|dkr| tj | | | jjj|| j|dS t|rJ| | | jjj|tdd| j|dS || jjj v }|rl| jjj | S i }t!|tj"tj	j
tj#j$j%tj#j&j'fv st(|rd }nt!|}| tj) t|dkr| tj | | t|tj"r|j*rt|tj+j,j-j.st/ddd	g d
 t|tj"rZt0|rZ| jj1rDtj2sZt/dddg t3j4d
 t5|rt6|d ur|j7t6|j7krt/dd|j7 dt6|j7 dg t3j4d
 t8|| jd|d}	| jjj9j:tdd| jt!||	|d}
t;| j|
| t<f | j|
|	||d|}|= r6t8|j>| jt?|ddd tj@}t|tArXtB|rXtjC}| tDjE|t|tFrr|ntG|d t(|r| tjH | tj) tItJ|KtjL |M \}}|D ]2}tN||}t?| jO|}tPQtR| j|| q|| jjj |< d|
jSjTvsJ tU||
jSjTd< |jVjSjTd }tW|| jjXurRtYdtZ||d|}||
jSjTd< |S )NrG  Tr   Zguardedr  r  z&Attempted to wrap strided NestedTensorrg  3torch.compile does not support strided NestedTensorrh  zAttempted to wrap sparse Tensorz-torch.compile does not support sparse Tensorsz.dtype mismatch between tensor and its gradientztensor dtype: z; grad dtype: zmInconsistent dtype between tensor and its gradient. This can happen in FSDP and crashes meta tensor creation.r  r   r   )r  r.  r4  r5  r   _base)r  r   r   ra  r8  r4  z(Wrapped Tensor must be this graph's fakeFr  )[r/  r  r  r$  r_   r*   rV  r   r   r;  r<  r;  r<  rJ  re   r\  rH   rI   rK   r  Zprepare_freezingis_specialized_nn_moduleZis_fsdp_modulerm  r_  r  r%  r3   rb  rG   rL  r  r>  r.  r   r	  r   r   r=  r>  r%   re  	is_nestednested	_internalnested_tensorNestedTensorr2   r   rS  Zcapture_sparse_computer+   r  r   r   r  wrap_to_fake_tensor_and_recordr  r  cache_real_value_when_exportwrap_fx_proxy_is_viewrp  r=   r?  rF   rJ   ZNOT_NONE_MATCHr  r  rN   r(   ZTENSOR_SUBCLASS_METADATA_MATCHr4   rR   r0  r  __tensor_flatten__r  r   r   r  r#  r  r  rV   r.  r   r!  r0   r   )r   r+  r   Zis_static_inputZshould_install_free_tensorZmake_graph_attributeZis_duplicate_tensoroptionsr5  r4  Ztensor_proxyr@  Z
guard_typeattrsr  attrZinner_valueinner_sourcefake_tensor_valuer  r   r   r   r?  J  sL   




	




	



	


zVariableBuilder.wrap_tensorc                 C   s  t d usJ t|t jsJ t|  }ddlm} |jj }|rtzd|j_W n$ t	yr   t|j
t jsnJ Y n0 t r z&||}|rddlm} ||}W n: ty } z"tdt|dg |d W Y d }~n
d }~0 0 W d    n1 s0    Y  tt| j|| t|| jd|d	}| jjjjtd
d| jt|||d}	t | j|	| d|i}
t!f t"| j|	|d|
}|| jjj#|< |j$j%j&d }t'||d|d|d}||	j%j&d< | j(|_(|S )Nr   )_utilT)clone_preserve_stridesz)failed to convert numpy.ndarray to TensorzHException encountered when attempting to convert numpy.ndarray to Tensor)ri  rj  rk  rl  Zfrom_excFro  r  r  r   r   r3  r  r.  r4  r4  r   r   r   r   r  ))rY  r   r[  rN   r/  Ztorch._numpyr  flagsZ	writeable
ValueErrorr  Znditerr   Ztemp_restore_stackZ_try_convert_to_tensorZtorch._prims_commonr  r4  r2   rR  r   r  r#  r  rw  r  r  r  rL  r  r  r.  rx  r=  r   r>  r.  r  r  r   r   )r   r+  r   r  readonlyZtensor_valuer  ra  r4  r.  r|  Znumpy_ndarray_variabler  r   r   r   r\  =  sx    


:
z"VariableBuilder.wrap_numpy_ndarrayr  c                 C   s  t |tu sJ | j| jjjv r0| jjj| j S | jjj}t j	rr|
 }t| | jjjt|| jd  nt|  s|d u rtjjjr| tj tj|| jdS | j }t| j|t|| j  d}t | j }| j}	t!|	t"r|	# }	|d ur|}
ntj$r8|j%t&u r8t'dd t( }
nrt!|	t)rl|	j*d urlt+|	j*|ddid sttj,s|t-j.}
n.|j%t&u rt'dd | tj tj|dS |j/|| j|
d}| jjjt|| j| n.t|  sJ | tj tj|| jdS t!|  t0rJ t1|  2tj3 d	|  i}| jjj4j5t67d
d| jt |||  d}|j8j9}t!|t:j;sJ | d|| jjj4j<|< t=||fi |}|| jjj| j< t|  st>|  |dd d|d|j8j?d< |S )Nr  r<  dynamo.automatic_dynamic_shapesTr   Fra  )r   r  r   r  r  r   r  r  r  )@r.  r  r  r  r  unspec_variable_mapr  r   r  Z#force_unspec_int_unbacked_size_liker   r   r	  rZ  r!   r   rG   r/  r   r  r*   r9  r%  r3   r:  r|   rs  r:   r8   make_scalarr;  r<  r   r   r?   rU  automatic_dynamic_shapesscalarr6   r   $get_automatic_dynamic_shapes_mark_asrM   r  r~  assume_static_by_defaultr   r  r  rP   r4   r0  re  r  r  rL  r  r  r  r  r  bound_symbolsr   r   r  )r   r+  r  rj  r  wrapped_valuer  frame_state_entrynormalized_source_namebase_sourcer  r|  r.  r,  
unspec_varr   r   r   r&    s    








	zVariableBuilder.wrap_symintc                 C   s  | j | jjjv r | jjj| j  S t| j| j  t|| j 	 d}t
jjjst|  st|st|st
jjjjstddstjr|jtur| tj tj|| jdS t
j|t
j d}t
j!j"#|r| tj tj|| jdS t$|  t%rJ t&|  'tj( t)|  }||d}t*|| jd|d}| jjj+j,t-.dd	| j t/|||d
}t0| j|| t1t2f| j||d|}t$|t2sJ || jjj| j < | jj3rt$|  t4st5d| d|   d }	|j6j7j8d }t9|s J |}	|	j:| jj:u s&J d|	j: dt;|  |d|	d|d|j7j8d< t<| j| jjj=ddgt>|gi R  }
| jjj?@tA|
jB| jd  tC jDdddd |
S )Nr  z.pytorch/compiler:unspecialize_float_killswitchFr  )r  )r   	raw_valuero  r  r  r   r  r.  r4  =Dynamo attempts to add additional input during export: value=	, source=r4  fake mode (^) from fake tensor metadata doesn't match mode({self.tx.fake_mode}) from InstructionTranslatorTr  r  call_methodr1  Ztensorify_float_attempt)	overwrite)Er  r  r  r  r:   r   r8   r  r;  r<  r   r  r*   ri  rG   r/  mathisnanisinfZ	_inductorZtritonZ
cudagraphsr   r  r  r6   r%  r3   r:  r|   rs  tensorfloat64r  r  Zis_gradtrackingtensorr   rP   r4   r0  re  rD   rw  r  r  rL  r  r.  rx  r=  r   rS  rM   AssertionErrorr.  r  r  r   r!  r   ry  r  rl   r	  rZ  r!   Zsym_numr
   rl  )r   r+  r  r  r   r|  r4  r.  r  r  r   r   r   r   rk  	  s    








zVariableBuilder.wrap_symfloatc                 C   s  | j | jjjv r | jjj| j  S t|}t|  tsLt	|  
tj d|  i}|d|i t|| jd|  d}| jjjjtdd| j t|||  d}t| j|| ttf| j||d|}|| jjj| j < t|  s| jjr t|  ts td	| d
|   d }t|tr8|j}n|jjjd }t |sTJ |}|j!| jj!u szJ d|j! dt"|  |d|d|d|jjd< |S )Nr   r  Fro  r  r  r   r  r  r  r4  r  r  Tr  r  )#r  r  r  r  r   r  r   r/  rP   r4   r0  r3   re  updaterw  r  r  rL  r  r.  rx  r=  r   rG   rS  rM   r  r|   r+  r.  r  r  r   r!  r   )r   r+  r  r|  r4  r.  r  r  r   r   r   r  	  sd    

z,VariableBuilder.wrap_unspecialized_primitive)NN)?r   r   r   __doc__rQ   r  r-  r(  r/  r%  classmethodr:  r  cacher7  rL  rM  rN  rO  rP  rQ  rT  rU  rX  r~  r  r   r   rz   r  r'  r  r   r@  rA  rk   r   rE  rp   rF  rm   rG  rH  rI  rJ  r   r   r  rJ  r;  r  r  rK  rm  r?  r\  r   r   r    r&  rk  r  r  r   r   r  r   r#    sh   	
&!       b 1 tU  w r#  c                 C   s   t | tr| j}n(tdd|  dt|  d|  dg d g }t|D ]8}d }| jrnt| jd}t	||j
}|t||d qHt|S )	Nzdataclass fields failurezobj: z; variable type: z$Dataclass fields handling fails for z*. Expected it to be a user-defined object.rh  Z__dataclass_fields__r   )r   r   r+  r2   r.  r  r  r   r=   rB   r  rZ  r   )objr+  r  fieldr   Zbase_srcr   r   r   r  	  s     

r  c                 C   s>   t | tjr:t | ts:t| r,t| |u s:| js:t| } | S r   )r   r   r   r   Z_is_functional_tensorr   rr  rY   )r+  r!  r   r   r   _clone_input	  s    

r  r  c                 K   sP   | |||d|}|d u r,t f dti|S t f dti|}||  |S d S )N)r  r.  r4  r5  r3  )r=  r   r   Zinstall_global)r  r.  r4  r5  r|  kwargsr^  r   r   r   ry  	  s    
ry  c                 C   sJ   | j rFtj & t|| j|jj|j< W d    n1 s<0    Y  d S r   )	rS  r   r  DisableTorchFunctionSubclassr  r!  tracerreal_value_cacher  r  r   r   r   rx  
  s
    rx  c                 K   sZ   |d u r t | ||||fi |S t|tjrDt| ||||fi |S t|||||| S d S r   )_wrap_fx_proxyr   r   r   _wrap_fx_preexisting_tensorhandle_traced_output)r3  r  r.  r4  r5  r|  r   r   r   r=  =
  s    

r=  c              	   K   s  ddl m} t|tjs*J dt| t||s8J d|v r^|d d ur^|jj|d  |j	j
dkrd|j	jv sJ d| dnd|j	jvsJ |j	jd  tjj  t||ju rnt||| |jrtj & t||j|jj|j	< W d    n1 s0    Y  d	| ttfv i}d
|v rB|d
 d usFJ |d
 |d
< t|fd|i|}|jjdkrt||jurtd| W d    n1 s0    Y  t| |||||S )Nr)   InstructionTranslatorBasez1_wrap_fx_preexisting_tensor expected tensor, got r5  placeholderr4  zplaceholder z* doesn't have 'example_value' in node.metar   r   r  r  zN`tensor` needs to be a `FakeTensor`wrapped by this instance of Dynamo. Found: )symbolic_convertr  r   r   r   r.  r  r5  r  r  opr  r  rR  +_disable_saved_tensors_hooks_during_tracingr   r!  rx  rS  r  r  r  r  r  r   r   rw  r  r0   construct_tensor_variable)r3  r  r.  r  r5  r|  r  r  r   r   r   r  R
  sR    
.$r  c                 K   s   ddl m} t||sJ d|v r@|d d ur@|jj|d  d|jjvs^J |jjd  tj	j
   t|j|dd}W d    n1 s0    Y  t|||||| S )Nr)   r  r5  r4  T)Zallow_non_graph_fake)r  r  r   r  r5  r  r  r  r   r  rR  r  r\   r  )r3  r  r.  r4  r5  r|  r  r   r   r   r  
  s    .r  c                 C   s  dd l }dd l}dd l}t| |jrNt|||| ||}|jjj||t	d |S t
|jjdr~|jjjdkr~t|jjj|jjs|jj|jjkrt|jjS |jj|jjks|jj|jjkrt| S t| |jrtdd | D rdd | D }t|fi |S t| ttfr|t|j|  g }	t| D ]\}
}|d u rT|	 t!j"d%i | nz|j#j$d	t%j&||
fi d
}d|v rt| tsJ |d }|' }t(||
dd|d< n|}|	 t)f ||||d| q*t| |jrt|	|fi |S t| trt*|	fi |S t| tt+fr.t,|	fi |S | j-j.dksdt
| dsdJ d| j-j. dt/|  t0|	| j-fi |S n| d u s|jj|j1u rt!j"d&i |S t| |j2|j3|j4fr|jj56| | t|j|  t7|| fi |S t89|jjrt:|jj|j;s(|jjdd t< D v rJt|j|  t=|| | j>fi |S t89|jjrlt:|jj|j?s|jjdd t< D v rt|j|  t@|| fi |S |jjdkr|jjAdkrt|j|  t!| fi |S | d ur.t| |j?r.|jjdkr.|jjAdkr.t|j|  t@|| fi |S t| tBr|jj|jCtDt%j&|jEjF|jGt%jH|jIjJjKtD|jLdtMtD|jLdtM|jNf
v s|jjAdkr|jjdv rt|j|  t!j"| fi |S t| |jOjjPrddlQmR} t|j|  ||fi |S t| tSrd|jj|jjT|jjIjU|jOjjV|jOjjW|jOjjXdgttYZ  v rdt|j|  t!j"| fi |S t| tBt[tSfr|jjt\u rt|j|  t!j"| fi |S t| t[s|jjdv rt|j|  t!j"| fi |S t]dd t^|  d!|jjA d"|jj d#g d$ d S )'Nr   )Zmutation_type_clsr   Z	set_statec                 s   s   | ]}t |tV  qd S r   )r   r  r  r   r   r   r   
  s   z'handle_traced_output.<locals>.<genexpr>c                 S   s   g | ]}t |qS r   r2  r  r   r   r   r2  
  r  z(handle_traced_output.<locals>.<listcomp>r  )kindtargetr  r  r   Fr6  r  ztorch.return_types_fieldsz	expected z. == torch.return_types or named tuple but got c                 S   s   g | ]\}}|j qS r   )Zcurrent_streamr  r  device_interfacer   r   r   r2    s   c                 S   s   g | ]\}}|j qS r   )r  r  r   r   r   r2    s   queryr  Zrecord_eventZget_rankZget_world_size)
bit_lengthru   r   
is_integer)hex	__round__ztorch.* op returned non-Tensorzexample_value type: z; op: z
; target: zUtorch.* ops that return a non-Tensor cannot be traced into the Dynamo FX graph outputrh  )N)N)_Ztorch._functorch.vmaptorch._subclasses.fake_tensorZtorch._utilsr   r   r  r  r$  Z
_track_objrv   r   r  r  r   r  r  	Generatorr  Zset_rng_stater   Z_DisableFuncTorchr  Z_is_in_bad_forkr   ri   rD  r|  r   r@  rA  rn   r  rZ  r|   rs  r  r  operatorgetitemrA  rE   r=  r   r#   r   r  r   r.  r   Zmanual_seedr	   ZSymFloatr  current_tracertrack_unbacked_symbolsr   r  r  r  r  r/   r   r  r  r   r  r  Zsym_intr  _utilsZ_element_sizeseedmodr  ZvmapZ_validate_and_get_batch_sizer^  r   Z_constrain_as_sizebackendsZ
SDPAParamssdpar   r  Z _are_functorch_transforms_activeZis_batchedtensorZis_flash_attention_availableZcan_use_flash_attentionZcan_use_efficient_attentionr   r}  rj  r   r2   rw   )r4  r  r.  r|  r5  r3  r   r*  sizesunpackedr  r'  Zproxy_ir   Z	options_ir   r   r   r   r  
  sR   


	







 r  c                 C   s   t ||j}t|j| |jj|| | |}t|t	j
jjr|j|ju r|rX|}n6t|t	jjrpt	jj}nt|t	jjrt	jj}nt	j}||d< || | |fi |S )z
    Actually construct a tensor variable after all the pre-processing from
    wrapping a pre-existing or newly created tensor value.
    Z
class_type)r  r!  rn   r  r  r  r  Z
specializer   r   r	  r   r   r;  r<  r  r   r  )r3  r  r.  r4  r5  r|  Zspecialized_propsZtensor_typer   r   r   r  k  s$    





r  c                   C   sD   t jdkrtjS t jdkr tjS t jdkr0tjS tdt j d S )NZdynamicZunbackedZ	obliviousz+invalid automatic_dynamic_shapes_mark_as = )r*   Z automatic_dynamic_shapes_mark_asr   r  rh  ZOBLIVIOUS_SIZEr  r   r   r   r   r    s    



r  _DYNAMIC_SOURCES_DYNAMIC_SOURCES_CONFIG_HASHc                  C   sL   t tjjj} td ur"t| kr"tS dd tjjjdddD a| atS )Nc                 S   s   h | ]}|r|qS r   r   r  sr   r   r   	<setcomp>  s   z&get_dynamic_sources.<locals>.<setcomp> rg  ,)	hashr   compilerr*   dynamic_sourcesr  r  replacesplitZcurrent_hashr   r   r   get_dynamic_sources  s    r  )source_namer  c                 C   s<   t  }|D ],}|| ks"t|| r
td| |  dS q
dS )NzA%s was marked dynamic due to dynamic source allowlist pattern: %sTF)r  rL  matchr(  r)  )r  r  patternr   r   r   rf    s    rf  r   )r  r  ra  r  c                    s   |  }t|s|  | }d g| } fddt|D }|jtd i }|D ]6\}}	|	 }
|||||
< ||||
  t	|
 qTng }t
| |tt|t|S )Nc                    s   g | ]} | | fqS r   r   r  Z	ex_strider   r   r2    r  z,record_automatic_dynamic.<locals>.<listcomp>)ru  )sizer   stridedimrI  sortr   r  
setdefaultr9   r:   r8   Zmake_tensorr@  )r  r  ra  Zex_sizer  r  pending
candidatesZi_strideZneg_ir  r   r  r   record_automatic_dynamic  s     
r  _UNBACKED_SOURCES_UNBACKED_SOURCES_CONFIG_HASHc                  C   sL   t tjjj} td ur"t| kr"tS dd tjjjdddD a| atS )Nc                 S   s   h | ]}|r|qS r   r   r  r   r   r   r    s   z'get_unbacked_sources.<locals>.<setcomp>r  rg  r  )	r  r   r  r*   unbacked_sourcesr  r  r  r  r  r   r   r   get_unbacked_sources  s    r  c                 C   s<   t  }|D ],}|| ks"t|| r
td| |  dS q
dS )NzC%s was marked unbacked due to unbacked source allowlist pattern: %sTF)r  rL  r  r(  r)  )r  r  r  r   r   r   rg    s    rg  Fc           -   
      s  | j r(t| tjjjjs(tdddg d | }|j	j
j| d }|rL|jnd }d }|  rvt|d}	t| j||	|}t| r|st| |||dd}
i }t| | \}}|D ]*}t| |}t||}t||||||< qt|
j|
j|
j|
j||
j|
j|dS |r\t|s\t|||  ttj g| !  tj"g| !  d g| !  d g| !  |||d	S d
dl#m$ t%fdd| & D rtdd | & D tj"g| !  d g| !  d g| !  |||d	S t||| }t'| }i   fdd}d
dl(m)} |j	j*r@|j	j*D ]4}t||rq
|j+|kr
||j!|j,|j q
g }g }g }g }g }t-| ! D ]jt| dt. v }t| dt. v }t| dt. v }t| dt. v }t| dt. v }|/t| di g  t0| } |}	t|	t1r|	2 }	|s8t|	t3rj|	j4d urjt5|	j4| di rjt67d| t8g| !  }!t9|!< |t:j;|!dO }t<j=oz|>}"t<j=o|?}#t|rt67d| d}"d}#t@|rt67d| d}"d}#|"p|#}$ }|d u rd }%d }&|r|t<jAs|d }&tB| drpfdd| jCD D }'|'jEd u rJ|'jFd u rJtGdd }%n$d
d!l#mH}( |(tI|'jE|'jFd"dd#}%n
tGdd }%nj|rtGdd }%nX|s|$rtJd$d |"rtGdd }%|#rtGdd }&n |st<j=stJd$d d }%d }&n*|\}%})d }&| d% d&}*|)|j	jKjL|*< |/|% |/|& |s6t@|r>tjM}+n`|%d usf|sf|sf| &  r||$rttN }+ntjO}+n"|st<jPs|rtj }+ntjQ}+|&d urtjO},ntj"},|/|+ |/|, q`t||||||||d'S )(NzGEncountered strided NestedTensor in automatic dynamic dim determinationrg  rn  rh  rp  T)
outer_only)dynamic_sizesdynamic_stridesconstraint_sizesconstraint_stridesview_base_contexttensor_source#shape_env_to_source_to_symbol_cacheinner_contexts)r  r  r  r  r  r  r  r   is_nested_intc                 3   s"   | ]}t |to | V  qd S r   )r   r	   r  r  r   r   r   G  r  z%_automatic_dynamic.<locals>.<genexpr>c                 S   s"   g | ]}t |trtjntjqS r   )r   r	   r   r  r%  r  r   r   r   r2  I  s   z&_automatic_dynamic.<locals>.<listcomp>c                    sZ   |  v rJddl m}  |  \}}||j|j@ dd}|p:|}||f | < n||f | < d S )Nr   StrictMinMaxConstraintFvrr  )%torch.fx.experimental.symbolic_shapesr  r  )r  constraint_ranger  r  Zold_constraint_rangeZold_nameZnew_constraint_rangenew_name)dim2constraintr   r   update_dim2constraint]  s    
z1_automatic_dynamic.<locals>.update_dim2constraint)_RelaxedConstraintZ_dynamo_strict_unbacked_indicesZ_dynamo_unbacked_indicesZ_dynamo_dynamic_indicesZ_dynamo_weak_dynamic_indicesZ_dynamo_static_indicesZ_specialize_onFz#automatic dynamic %s marked dynamic)r  rc  rd  _dynamo_dynamic_rangec                    s   g | ]}|j  kr|qS r   )r  )r  Zdr)r  r   r   r2    s   r  r  )lowerupperr  r  z.size()[])r  r  r  r  specialize_onr  r  r  )Rrr  r   r   rs  rt  ru  rv  r2   r  r  tracing_contexttensor_to_contextr  r  rz  r=   _automatic_dynamicrp  r%   r.  r{  r  r   r  r  r  r  r  rf  r  r   r   r%  r  ZINFER_STRIDEr  r  anyr  r  r$  r  Zexport_constraintst_idr  rI  rl  rZ  r   r?   rU  rM   r  r~  r(  r)  r7   r6   r8   Z	make_sizer*   r  Zis_size_dynamicZis_stride_dynamicrg  Zallow_ignore_mark_dynamicr   r  popminmaxr   r  r'   r   r  Zsource_name_to_debug_namerh  r  r  r  ZDUCK)-ra  r  r   static_shapesr  r  Zprior_policyr  r  r  Zouter_contextr  r}  r  r~  Zinner_tensorr  r  r  r  r  r  r  r  r  r  r  Zmarked_strict_unbackedZmarked_unbackedZmarked_dynamicZmarked_weak_dynamicZmarked_staticr  Z	mark_sizeZautomatic_dynamic_sizeZautomatic_dynamic_strideZautomatic_dynamicZconstraint_sizeZconstraint_strideZ	dim_ranger  Zname_Zdim_nameZdynamic_sizeZdynamic_strider   )r   r  r  r   r
    sr   





	











	

r
  )parent_context)r   r   c                   s  t  tjtjjtfv s.t tjs.t rd us:J t |d\}}|s`t	 |ntt
snJ j}|j| td t jt   t fdd}d urt|tr|j }	d urjjt|	t t|rH| \}
}|
D ]>}t||}t |}t
|}t||t|tjd qjjj < t|r|jrp|  n|! }|" d|j# |" |$ djj%< n|" |$ djj%< |r|r҈& st'sjjt| jj(t)   |S  S d S )	N)r  zwrap_to_fake %s %s %s %sc                      s   j j dS )N)r   symbolic_context)r!  r  r   ra  r   r  r  r   r   r  6  s   z0wrap_to_fake_tensor_and_record.<locals>.<lambda>)r   r   r  )ru   )r  r  Zvalues_sizeZvalues_stride)r  r  )*r.  r   r   r;  r<  r   r   r%   ro   r
  r=   r  r  r(  r)  r  r@  shapert   Z	item_memor  r	  rZ  r!   r>   r{  r  rw  r  r	  r   Z	is_sparseZ_valuesvaluesr  ndimr  Zinput_source_to_sizes_stridesrq  rG   Ztracked_fakes_id_to_sourcer  )ra  r  r   r   r  r  Z_reasonZinner_context_nameZfake_eZsym_valr}  r  r~  Z
fake_innerinnerr  r  r   r  r   rw    s    

	









rw  c                   @   sJ   e Zd ZdZddddZededdd	Zed
d Zedd Z	dS )r  a  
    Like builder, but stateless and does not require a source. Useful for simple type->VT objects, or objects
    that are being created/evaporated during inlining (ex: consider a locally made list of tensors we then iterate over
    .), such a list should not show up as an artifact from inputs, nor in reconstruction, nor in the graph. However,
    there may be reasons to represent it as a ListVariable internally.

    NOTE - Objects produced here are born UNGUARDED due to the nature of sources!

    NOTE - This class is very new! It will have some rough edges, but it was created to stem the bleeding of giant
    if/else type->VariableTracker trees that were cropping up all over dynamo.
    Nr  c                 C   s   t dd S )NzUse SourcelessBuilder.create()r  r   r   r   r   r    s    zSourcelessBuilder.__init__r   r  r  c                    s  t }tj|}|r"| S ttr0S ttjrDtS t	
rXt	S trtd urtrd j_tS trtS ttjtjtjjjfrtS tt tjfrtS ttj  rt!S ttj"rjtj#t tjfrjt$j#j%j&ks0J t j#}z|' j%j&W S  t(yd   Y n0 n^ttj)j*j+rt,S ttj-j.j/tj-j.j0frtS t12rt1S t34rt3S t5j6u rt7S tt8j9rt:S ttj;j<j=j>rt	t?S tt tj@jAr@tj;j<jBCS ttjDrVtES tFr fddtGt D }tH|t dS ttjIrȈjJjK jjLv rȈ jjLjJjK }tM |S tNd|jO d|jP d| g tQjRd d S )	NTc                    s   g | ]}t  t|qS r   )r  rs  r  r  r  r+  r   r   r2    s   z,SourcelessBuilder.create.<locals>.<listcomp>)r  z%Unexpected type in sourceless builderrL  z3SourcelessBuilder.create does not know how to wrap rh  )Sr.  r  _type_handlersr  r   rz   r  Z_HAS_DEFAULT_FACTORY_CLASSr   r|   ro  rs  r  r.   r  r  r  r  ra   r&  r  r  r   r  r  r  r  r}   abcABCMetar   rV  r  r   r  r  r  r  r   Zvar_getattrr4  rT  Zgraph_moduleZGraphModuler   rR  r  r  ZLeafSpecr   r  r   r  r  r  r   rL  rM  r   r  	variableslazyZLazySymNodeFormatStringrR  r  Zflex_attention_backwardhigher_order_opsZ&FlexAttentionBackwardHighOrderVariableGenericAliasr   rd   rj   r   r	   r  r  r  r   r2   r   r   r+   Z
DYNAMO_BUG)r  r+  
value_typeZfast_handlerZ
cls_obj_vtr  r.  r   r  r   rs    s    










zSourcelessBuilder.createc                 C   s   t | sJ t j| dS )Nra  )r|   ro  rs  ra  r   r   r   wrap_constant_literal  s    z'SourcelessBuilder.wrap_constant_literalc                     s2  t j i } tD ]}dd | |< q fdd| t<  fdd| t<  fdd| t<  fdd| t<  fdd| tj< | t | t	j
< | t | t< | t | t< dd | tj< d	d | tj< d
d | tj< dd | tjjj< dd | tjjj< dd | tjjj< dd | tjjj< dddd}tjD ]}|| |< q| S )Nc                 S   s   t |S r   )r|   r  r   r   r   r    r  z6SourcelessBuilder.make_type_handlers.<locals>.<lambda>c                    s   t  fdd|D t dS )Nc                    s   g | ]} |qS r   r   r  rs  r  r   r   r2    r  JSourcelessBuilder.make_type_handlers.<locals>.<lambda>.<locals>.<listcomp>r  )r   ry   r  rs  r  r   r    s   c                    s(   t  fdd| D t|t dS )Nc                    s"   i | ]\}} | |qS r   r   rz  r$  r   r   
<dictcomp>  r  zJSourcelessBuilder.make_type_handlers.<locals>.<lambda>.<locals>.<dictcomp>r&  )r   r  r.  ry   r  r'  r(  r   r    s   c                    s   t  fdd|D t dS )Nc                    s   g | ]} |qS r   r   r  r$  r   r   r2    r  r%  r&  )r   ry   r  r'  r(  r   r    s   c                    s   t  fdd|D S )Nc                    s   g | ]} |qS r   r   r  r$  r   r   r2    r  r%  )r   r  r'  r(  r   r    s   c                    s   t  fdd|D S )Nc                    s   g | ]} |qS r   r   r  r$  r   r   r2    r  r%  )r   r  r'  r(  r   r    s   c                 S   s   t  S r   )r   r  r   r   r   r    r  c                 S   s   t |S r   )r   r  r   r   r   r    r  c                 S   s   t |t dS Nr&  )r   ry   r  r   r   r   r    s   c                 S   s   t |t dS r*  )r   ry   r  r   r   r   r    s   c                 S   s   t |t dS r*  r   ry   r  r   r   r   r    s   c                 S   s   t |t dS r*  r+  r  r   r   r   r    s   c                 S   s   t |t dS r*  r+  r  r   r   r   r    s   r   r(  c                 S   s   |S r   r   r  r   r   r   passthrough  s    z9SourcelessBuilder.make_type_handlers.<locals>.passthrough)r  rs  rZ   rl  r~  rA  r@  r   rD  rB  r  r"   r#   r  r  rV  r  r  r  r  r  distributionsconstraintsZ_RealZ	_Interval
Constraintr{   Zall_subclasses)handlersr   r,  r9  r   r'  r   make_type_handlers  s6    
z$SourcelessBuilder.make_type_handlers)
r   r   r   r  r  r  rz   rs  r#  r1  r   r   r   r   r  u  s   Q
r  c                   @   s2   e Zd ZdZddddZededdd	ZdS )
"SourcelessUserDefinedObjectBuilderz
    SourceLessBuilder does not return a UserDefinedObjectVariable, but in some
    cases it might be ok to return UserDefinedObjects. In such case, use this
    builder.
    Nr  c                 C   s   t dd S )Nz/Use SourcelessUserDefinedObjectBuilder.create()r  r   r   r   r   r  $  s    z+SourcelessUserDefinedObjectBuilder.__init__r   r  c                 C   sN   t |}t|tr t|t dS t|tjjr<t	|t dS t
|t dS d S r*  )r.  r  r   r   ry   r   r   r;  r  r   r   )r  r+  r"  r   r   r   rs  '  s    
z)SourcelessUserDefinedObjectBuilder.create)r   r   r   r  r  r  rz   rs  r   r   r   r   r2    s   r2  )NN)NN)N)NN)F(Y  r  r  rB  r  rA  r  r  r  r  	itertoolsr  r  r  r  rL  r  r  rV  r   rO  collections.abcr   typingr   r   r   r   r   r   r  r   r	   Ztorch._dynamo.utilsr
   r   r   r   Ztorch._guardsr   Z!torch._higher_order_ops.torchbindr   Z
torch._opsr   r  r   r   r   Ztorch._subclasses.meta_utilsr   r   Ztorch._utils_internalr   Z%torch.fx.experimental._backward_stater   Ztorch.fx.experimental._dynamismr   r  r   r   r   r   r   r   r   r    r!   Ztorch.fx.immutable_collectionsr"   r#   Z torch.nn.utils._expanded_weightsr$   Ztorch.utils._python_dispatchr%   r&   Ztorch.utils._sympy.value_rangesr'   Ztorch.utils.weakr(   rg  r*   r+   r,   r-   r.   r  r/   excr0   r1   r2   r5  r3   r4   r5   Zpgor6   r7   r8   r9   r:   r$  r;   r   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rR  rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   r  rv   rw   rx   ry   rz   r{   Zconstantr|   r}   Zctx_managerr~   r   r   r   r   r   r   Zdictsr   r   r   r   r   r   r^  r   r   r   r   r   Z	functionsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   iterr   r  r   listsr   r   r   r   r   r   r   r   r   miscr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Z	nn_moduler   r   r   Z	optimizerr   Zscript_objectr   r  r   r  r   r   r   r   r   r   r   r   r   r   Ztorch_functionr   r   r   Zuser_definedr   r   r   r   r   r   r   r   r   r   r   numpyrY  ModuleNotFoundErrorZtorch._dynamo.codegenr   Ztorch._dynamo.symbolic_convertr   	getLoggerr   r(  Z_loggingZgetArtifactLoggerrI  rA  ZDimListr   r   Z	dataclassr   r  r  varsr  r!  r  r  rl  r"  r;  r  rZ  r[  rW  rX  r#  r  r  ry  rx  r=  r  r  r  r  r  r  rR  r  r  r  rf  r   r  r  r  r  rg  r
  rw  r  r1  r  r2  r   r   r   r   <module>   s(   ,p! $	 <,`8("b

<
                C

 
8  B 
 A
%
   
e &