o
    Zhw)                    @  sp  U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$m%Z% ddl&Z&ddl'Z&ddl(Z&ddl)m*  m+Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZD ddlEmFZFmGZG ddlHmIZI ddlJmKZKmLZL ddlMmNZN ddlOmCZC ddlPmQZQ ddlRmSZS e	TeUZVe&jWjXZXe&jWjYZYeZZe"eZe&jMj[f Z\G dd de Z]G dd  d e Z^G d!d" d"e Z_e!d#Z`e"e&jMjajbecf ZdG d$d% d%Zeee Zf	&ddd/d0ZgG d1d2 d2ZhG d3d4 d4eiZje"ehejf Zkdd8d9ZlG d:d; d;ZmG d<d= d=eZnG d>d? d?enZoG d@dA dAenZpG dBdC dCenZqG dDdE dEenZrG dFdG dGenZsetedHf ZuG dIdJ dJesZvG dKdL dLevZwG dMdN dNevZxG dOdP dPevZyG dQdR dResZzG dSdT dTezZ{G dUdV dVezZ|G dWdX dXezZ}G dYdZ dZenZ~G d[d\ d\enZG d]d^ d^enZG d_d` d`ZG dadb dbe ZejG dcdd ddZejG dedf dfeZejG dgdh dheZejG didj djeZddmdnZddsdtZ	uddd|d}Zedd~ddufdddZeAec  ZdddZeejd d Zg Zded< edd~dufdddZeDjdud		~ddddZ		~dd ddZefduddddZefduddddZdddZedZdddZdddZdddZdddZdddZdddZG dd dZd	ddĄZ	~	~		~d
ddd˄Ze& ddd͜dddԄZe& dddքZdddڄZddd܄Zddd߄ZdddZdddZeAec  Z	ddddZdddZdddZdS (  a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fxstatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                   @  s    e Zd ZU ded< d
ddZd	S )SearchFnstr__name__argsr   kwargsreturnc                 O     d S N selfr0   r1   r5   r5   N/var/www/auris/lib/python3.10/site-packages/torch/_inductor/pattern_matcher.py__call__]       zSearchFn.__call__Nr0   r   r1   r   r2   r   )r/   
__module____qualname____annotations__r9   r5   r5   r5   r8   r-   Z   s   
 r-   c                   @     e Zd ZdddZdS )		ReplaceFnr0   r   r1   r2   c                 O  r3   r4   r5   r6   r5   r5   r8   r9   a   r:   zReplaceFn.__call__Nr;   r/   r<   r=   r9   r5   r5   r5   r8   r@   `       r@   c                   @  s   e Zd Zddd	Zd
S )TraceFnfnUnion[SearchFn, ReplaceFn]r0   r   r1   r2   torch.fx.GraphModulec                 O  r3   r4   r5   )r7   rD   r0   r1   r5   r5   r8   r9   e      zTraceFn.__call__N)rD   rE   r0   r   r1   r   r2   rF   rA   r5   r5   r5   r8   rC   d   rB   rC   Tc                   @  s   e Zd ZdddZdS )Multipler2   Nonec                 C  s   dt  vs| tu sJ d S d S )NMULTIPLE)globalsrK   r7   r5   r5   r8   __init__q   s   zMultiple.__init__Nr2   rJ   )r/   r<   r=   rN   r5   r5   r5   r8   rI   p   rB   rI    new_metadict[str, Any]old_nodetorch.fx.Node	pass_namer.   r2   rJ   c                 C  s   ddl m}m} tjjr1| dg  }|||||j	 | 
dd |j D  || d< d S | 
dd |j D  d S )Nr   )
NodeSourceNodeSourceActionZ	from_nodec                 s  *    | ]\}}|t jjjv r||fV  qd S r4   torchfxproxyZ_COPY_META_FIELDS.0kvr5   r5   r8   	<genexpr>       z!_transfer_meta.<locals>.<genexpr>c                 s  rX   r4   rY   r]   r5   r5   r8   ra      rb   )Ztorch.fx.tracebackrV   rW   r#   traceenabledgetcopyappendZREPLACEupdatemetaitems)rQ   rS   rU   rV   rW   Znew_from_noder5   r5   r8   _transfer_metaz   s   

rk   c                      s   e Zd ZU dZded< ded< ded< ded	< d
ed< ded< ded< 		d6d7 fddZed8ddZd9ddZd:ddZ	d;d d!Z
d<d"d#Zd=d%d&Zd>d(d)Zd?d+d,Z		-d@dAd4d5Z  ZS )BMatcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]r0   rR   r1   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphNOptional[Sequence[Any]]Optional[dict[str, Any]]r2   rJ   c                   sD   t    || _t|pg | _|pi | _g | _i | _|| _d | _	d S r4   )
superrN   rn   listr0   r1   rq   rr   rt   ru   )r7   rt   rn   r0   r1   	__class__r5   r8   rN      s   


zMatch.__init__torch.fx.Graphc                 C     | j jS r4   )rt   graphrM   r5   r5   r8   r~         zMatch.graphotherc                 C  s   | j r#t| j  t|j  @ D ]}| j | |j | kr"td|q| j|j | j|j | j |j  | j|j d S )Nzkwarg mismatch: {})	r1   r    keysFailedMatchr0   extendrq   rh   rr   )r7   r   keyr5   r5   r8   r      s    
zMatch.extendc                 C  s"   | j rt| j g| _ | S g | _ | S r4   )r0   tuplerM   r5   r5   r8   bundle   s   zMatch.bundler.   c                 C  s   d| j  d| j dS )NzMatch(..., , )r0   r1   rM   r5   r5   r8   __repr__      zMatch.__repr__c                 C  s0   | j }t| jD ]}|js|js|| qd S r4   )r~   reversedrq   _erasedusers
erase_node)r7   r~   nr5   r5   r8   erase_nodes   s   
zMatch.erase_nodeslist[Optional[torch.fx.Node]]c                   s    fdd j jD S )Nc                   s$   g | ]}|d ur j j| nd qS r4   )rt   pattern_to_noder^   prM   r5   r8   
<listcomp>   s    z&Match.output_nodes.<locals>.<listcomp>)rt   outputsrM   r5   rM   r8   output_nodes   s   
zMatch.output_nodesrT   c                 C  s   t dd |  D S )Nc                 s  s    | ]}|r|V  qd S r4   r5   r   r5   r5   r8   ra      s    z$Match.output_node.<locals>.<genexpr>)nextr   rM   r5   r5   r8   output_node   r   zMatch.output_nodeSequence[Any]c                 C  s   t | | jj|| d S r4   )ReplacementPatternEntryreplace_with_graphrt   r~   )r7   ru   r0   r5   r5   r8   r      s   zMatch.replace_with_graphTreplacement_fnr@   trace_fnOptional[TraceFn]run_functional_passesboolc           
      C  s   ddl m}m} t|j|r|jdu r|jnt }|C |du r(tjt	|d}||t
j|dd }t| jdkrM|jjD ]}	t|	j| jd dd	 q?t| | jj|| W d   dS 1 sbw   Y  dS )
a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVN)r   c                 S  
   | j d S Nvalri   )argr5   r5   r8   <lambda>     
 z*Match.replace_by_example.<locals>.<lambda>r*   replace_by_examplerQ   rS   rU   )Ztorch._inductor.virtualizedr   r   
isinstanceZ	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyrZ   r[   map_arglenrq   r~   rk   ri   r   r   rt   )
r7   r   r0   r   r   r   r   contextreplacementr   r5   r5   r8   r      s:   

"zMatch.replace_by_example)NN)
rt   rs   rn   rm   r0   rv   r1   rw   r2   rJ   )r2   r|   )r   rl   r2   rJ   )r2   rl   r2   r.   rO   )r2   r   )r2   rT   )ru   r|   r0   r   r2   rJ   NT)
r   r@   r0   r   r   r   r   r   r2   rJ   )r/   r<   r=   __doc__r>   rN   propertyr~   r   r   r   r   r   r   r   r   __classcell__r5   r5   rz   r8   rl      s0   
 







rl   c                   @  s8   e Zd ZU dZded< dd	d
ZdddZdddZdS )r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r.   format_stringr0   r   r1   r2   rJ   c                 O  s0   || _ t|dkrtd| || _|| _d S )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr0   r1   )r7   r   r0   r1   r5   r5   r8   rN      s   
zFailedMatch.__init__c                 C  s   | j j| ji | jS r4   )r   formatr0   r1   rM   r5   r5   r8   __str__+  r   zFailedMatch.__str__r   c                 C     dS NFr5   rM   r5   r5   r8   __bool__.     zFailedMatch.__bool__N)r   r.   r0   r   r1   r   r2   rJ   r   r2   r   )r/   r<   r=   r   r>   rN   r   r   r5   r5   r5   r8   r     s   
 

r   mMatchResultTypeIs[Match]c                 C     t | S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )r   )r   r5   r5   r8   is_match5  s   r   c                   @  sT   e Zd ZU dZded< ded< ded< ded	< 	
ddddZdddZdddZd
S )rs   zC
    Internal state needed while running PatternExpr._match().
    list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r|   r~   zlist[NodeOrConstant]exclusive_node_setN*Optional[dict[PatternExpr, torch.fx.Node]]r2   rJ   c                C  s,   || _ |d u r	i nt|| _|| _g | _d S r4   )r   dictr   r~   r   )r7   r   r   r~   r5   r5   r8   rN   G  s   
zMatchContext.__init__rn   rm   nodeNodeOrConstantr   c                 C  sZ   || j v r| j | |krt| |S tdS ||| }|| j vs"J |r&|nd| j |< |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   rl   r   _match)r7   rn   r   r   r5   r5   r8   matchS  s   

zMatchContext.match dict[PatternExpr, torch.fx.Node]c                 C  s   dd | j  D S )Nc                 S  s&   i | ]\}}|  r|d ur||qS r4   )has_multiple_users)r^   rn   r   r5   r5   r8   
<dictcomp>`  s    z;MatchContext.filter_multi_user_patterns.<locals>.<dictcomp>)r   rj   rM   r5   r5   r8   filter_multi_user_patterns_  s   z'MatchContext.filter_multi_user_patternsr4   )r   r   r   r   r~   r|   r2   rJ   )rn   rm   r   r   r2   r   )r2   r   )r/   r<   r=   r   r>   rN   r   r   r5   r5   r5   r8   rs   =  s   
 
rs   c                   @  sP   e Zd ZdZeddd	Zdd
dZdddZdddZd ddZ	d!ddZ
dS )"rm   z+
    Base class for types of patterns.
    r   rT   rt   rs   r2   r   c                 C  r3   r4   r5   r7   r   rt   r5   r5   r8   r   l  r   zPatternExpr._matchc              
   C  sD   zt | g|jd| |W S  ty! } z|W  Y d }~S d }~ww Nr~   )rs   r~   r   r   r7   r   er5   r5   r8   r   o     zPatternExpr.matchr   c                 C  r   r   r5   rM   r5   r5   r8   r   u  r   zPatternExpr.has_multiple_usersr.   c                 C  s   | j jd S )Nz())r{   r/   rM   r5   r5   r8   r   x  s   zPatternExpr.__repr__searchedOrderedSet[torch.fx.Node].Generator[Optional[torch.fx.Node], None, None]c                 c  s     | |j v r|j |  V  d S d S r4   )r   r7   rt   r   r5   r5   r8   find_anchor_nodes{  s   
zPatternExpr.find_anchor_nodesr   r   c                 C  s   t || jS )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   r{   r7   r   r5   r5   r8   
pattern_eq  s   zPatternExpr.pattern_eqNr   rT   rt   rs   r2   r   r   rT   r2   r   r   r   rt   rs   r   r   r2   r   r   r   r2   r   )r/   r<   r=   r   r   r   r   r   r   r   r   r5   r5   r5   r8   rm   g  s    



rm   c                   @     e Zd ZdZddd	Zd
S )Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    r   r   rt   rs   r2   r   c                 C  s   t || |gdS )N)r0   rl   r   r5   r5   r8   r     s   z
Arg._matchNr   r   rt   rs   r2   r   r/   r<   r=   r   r   r5   r5   r5   r8   r     s    r   c                   @  s.   e Zd ZdZddd	ZdddZdddZdS )Ignoredz4
    Match an arg, but don't pass it to handler
    r   r   rt   rs   r2   r   c                 C  s
   t || S r4   r   r   r5   r5   r8   r        
zIgnored._matchr.   c                 C  r   )N*r5   rM   r5   r5   r8   r     r   zIgnored.__repr__ppPatternPrettyPrinterc                 C  r   )Nz	Ignored()r5   )r7   r   r5   r5   r8   pretty_print  r   zIgnored.pretty_printNr   r   r   r   r2   r.   )r/   r<   r=   r   r   r   r   r5   r5   r5   r8   r     s
    

r   c                      sD   e Zd ZdZd fddZddd	ZdddZd fddZ  ZS )
KeywordArgD
    Capture a kwarg which will become an input to the handler.
    namer.   r2   rJ   c                      t    || _d S r4   rx   rN   r   r7   r   rz   r5   r8   rN        

zKeywordArg.__init__c                 C     d| j dS )NzKeywordArg(r   r   rM   r5   r5   r8   r        zKeywordArg.__repr__r   r   rt   rs   r   c                 C  s   t || | j|idS )Nr1   )rl   r   r   r5   r5   r8   r        zKeywordArg._matchr   r   r   c                   $   t t|}t |o| j|jkS r4   typingcastr   rx   r   r   r   rz   r5   r8   r        zKeywordArg.pattern_eqr   r.   r2   rJ   r   r   r   	r/   r<   r=   r   rN   r   r   r   r   r5   r5   rz   r8   r     s    

r   c                      sN   e Zd ZU dZded< d fddZddd	ZdddZd fddZ  Z	S )ExclusiveKeywordArgr   r.   r   r2   rJ   c                   r   r4   r   r   rz   r5   r8   rN     r   zExclusiveKeywordArg.__init__c                 C  r   )NzExclusiveKeywordArg(r   r   rM   r5   r5   r8   r     r   zExclusiveKeywordArg.__repr__r   r   rt   rs   r   c                 C  s2   ||j v r	tdS |j | t|| | j|idS )Nzexclusive arg appears twicer   )r   r   rg   rl   r   r   r5   r5   r8   r     s   
zExclusiveKeywordArg._matchr   r   r   c                   r  r4   r  r   rz   r5   r8   r     r  zExclusiveKeywordArg.pattern_eqr  r   r   r   )
r/   r<   r=   r   r>   rN   r   r   r   r   r5   r5   rz   r8   r    s   
 

r  c                      s   e Zd ZU dZded< ded< 	d)d* fddZeed+ddZd+ddZ	d+ddZ
d,ddZd-ddZd.d!d"Zd/d#d$Zd0 fd'd(Z  ZS )1_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setr*   !Union[FnsType, Sequence[FnsType]]r   Union[Multiple, int]r2   rJ   c                   sx   t    t|st|tr|gnt|}|D ] t tjjr.|	 fdd 
 D  q|| _t|| _|| _d S )Nc                 3  s    | ]}t  |V  qd S r4   getattrr^   overloadrD   r5   r8   ra         z'_TargetExpr.__init__.<locals>.<genexpr>)rx   rN   callabler   r.   ry   rZ   _opsOpOverloadPacketr   	overloadsr
  r    r  r   )r7   r
  r   rz   r  r8   rN     s   
 

z_TargetExpr.__init__r.   c                 C  r3   r4   r5   rM   r5   r5   r8   op  rG   z_TargetExpr.opc                 C  s   | j d }t|ts|j}t| j dkrd| dS | j d tt|d u r*d| S | j d tt|d u r:d| S t| j d tjj	rKt| j d S |S )Nr   r*   [z, ...]torch.z	operator.)
r
  r   r.   r/   r   r  rZ   operatorr  
OpOverload)r7   Z
first_reprr5   r5   r8   fns_repr  s   



z_TargetExpr.fns_reprc                 C  sJ   | j tu rd}n| j dkrd| j  d}nd}| jj d|   | dS )Nz
, MULTIPLEr*   r   r   rP   ()r   rK   r{   r/   r  )r7   Zcomma_usersr5   r5   r8   r     s   

z_TargetExpr.__repr__r   c                 C  s   t | jtp
| jdkS )Nr*   )r   r   rI   rM   r5   r5   r8   r     r   z_TargetExpr.has_multiple_usersrt   rs   r   r   r   c                 C     t r4   NotImplementedErrorr   r5   r5   r8   r     s   z_TargetExpr.find_anchor_nodesr   rT   c                 C  s(   t |tjjo|j| jkot|| jv S r4   )r   rZ   r[   Noder  extract_targetr  )r7   r   r5   r5   r8   
_match_fns  s
   
z_TargetExpr._match_fnsc                 C  s$   | |j v p| jtu pt|j| jkS r4   )r   r   rK   r   r   r5   r5   r8   _match_users  s
   
z_TargetExpr._match_usersr   r   c                   s<   t t|}t |o| j|jko| j|jko| j|jkS r4   )r  r  r   rx   r   r  r
  r   r   rz   r5   r8   r     s   


z_TargetExpr.pattern_eq)r*   )r
  r  r   r  r2   rJ   r   r   r   r   rT   r2   r   )r   rT   rt   rs   r2   r   r   )r/   r<   r=   r   r>   rN   r   r   r  r  r   r   r   r$  r%  r   r   r5   r5   rz   r8   r	    s    
 


	


r	  .c                      sz   e Zd ZdZddd- fddZed.ddZed.ddZd/ddZd0ddZ	d1d"d#Z
d2d'd(Zd3 fd+d,Z  ZS )4_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r*   )_usersr
  /Union[torch.fx.node.Target, str, Sequence[Any]]r0   r   r(  Union[int, Multiple]r1   r2   rJ   c                  sh   t  || t|| _t|| _tdd t||	 D r%| j
| _n| j| _| | j| j| _d S )Nc                 s  s     | ]}t |tttfV  qd S r4   )r   r   ry   r   r^   xr5   r5   r8   ra   5  s
    
z+_TargetArgsExpr.__init__.<locals>.<genexpr>)rx   rN   r   r0   r   r1   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r7   r
  r(  r0   r1   rz   r5   r8   rN   +  s   


z_TargetArgsExpr.__init__r   Mapping[Any, Any]9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]c                 C  s.   g | |  R }t| g| R }||fS r4   )r0  r   r   )r0   r1   r0  specr5   r5   r8   r3  >  s   z_TargetArgsExpr.simple_flattenc                   sR   t tttttid	 fdd tj | |ffddd}t|\}}||fS )
Nr,  r   r2   c                   s:   t | }|}|d urtj || fdddS | S )Nc                      t |  v S r4   typer,  type_mappingr5   r8   r   W      zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>Zis_leaf)r:  re   pytreetree_map)r,  clsZ
convert_fnconvert_typer=  r5   r8   rD  P  s   

z4_TargetArgsExpr.pytree_flatten.<locals>.convert_typec                   r8  r4   r9  r;  r<  r5   r8   r   ^  r>  z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>r?  )r,  r   r2   r   )r   r   ry   r   r   r@  rA  Ztree_flatten)r0   r1   Znormalized_args_treeZflatr7  r5   rC  r8   r1  F  s   
z_TargetArgsExpr.pytree_flattenr.   c                 C  sv   |   gtt| jdd | j D }| jtu r |d n| jdkr.|d| j  | j	j
 dd| dS )	Nc                 S  s   g | ]\}}| d | qS =r5   r]   r5   r5   r8   r   g  s    z,_TargetArgsExpr.__repr__.<locals>.<listcomp>_users=MULTIPLEr*   _users=r  r   r   )r  mapreprr0   r1   rj   r   rK   rg   r{   r/   join)r7   r0   r5   r5   r8   r   c  s   


z_TargetArgsExpr.__repr__r   r   c                   s   |   g fdd| jD  fdd| j D }| jtu r&|d n| jdkr4|d| j  d}| jj d	|	| d
S )Nc                 3  s    | ]}  |V  qd S r4   r   r+  r   r5   r8   ra   r  r  z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>c                   s$   g | ]\}}| d   | qS rE  rL  r]   rM  r5   r8   r   s  s   $ z0_TargetArgsExpr.pretty_print.<locals>.<listcomp>rG  r*   rH  r   r  r   )
r  r0   r1   rj   r   rK   rg   r{   r/   rK  )r7   r   r0   
joiner_strr5   rM  r8   r   o  s   

z_TargetArgsExpr.pretty_printr   rT   rt   rs   r   c                   s   |rt|jtjkrtd|S ||s tdS |j}|j t tjk rvddlm} t|j	s<J ||j	|j|j}|d u rOtd|S |\} t|tjkrpt tjkrp fdd D  ntd|S  fdd D  
| \}}j\}}	||	krtd||	S t|t|ksJ t|}
tt ||D ]1\}}}t|tr|||}t|s|  S |
| qt|tjjs||krtd	||  S q|
j| |j	|
j< |
S )
Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionc                       i | ]}|j v r| | qS r5   r   r^   i_kwargsr7   r5   r8   r          z*_TargetArgsExpr._match.<locals>.<dictcomp>c                   rP  r5   r   rQ  rS  r5   r8   r     rU  zargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r$  r   r0   r   r%  r1   Ztorch.fx.operator_schemasrO  r  targetr2  r4  rl   zipr.  countr   rm   r   r   r   rZ   r[   r"  rq   rg   rr   )r7   r   rt   _argsrO  Znormalized_args_and_kwargsZ
node_itemsZ	node_specZ
self_itemsZ	self_specr   rR  rn   
child_nodechild_matchr5   rS  r8   r   }  sR   
$


z_TargetArgsExpr._matchr   r   r   c                 c  s    | |j v r|j |  V  dS | jd D ]/}t|trB|||D ]!}t|tjjs*q |jD ]}||vr@| 	|r@|V  |
| q-q qdS )a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   r4  r   rm   r   rZ   r[   r"  r   r$  add)r7   rt   r   rn   Z
other_noder   r5   r5   r8   r     s$   





z!_TargetArgsExpr.find_anchor_nodesr   r   c                   sP   t t|}t |o'| jd |jd ko'tdd t| jd |jd D S )Nr*   c                 s  0    | ]\}}t |tr||n||kV  qd S r4   r   rm   r   r^   abr5   r5   r8   ra     
    
z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>r   )r  r  r   rx   r   r4  allrW  r   rz   r5   r8   r     s   z_TargetArgsExpr.pattern_eq)
r
  r)  r0   r   r(  r*  r1   r   r2   rJ   )r0   r   r1   r5  r2   r6  r   r   r   r   r   )r/   r<   r=   r   rN   staticmethodr3  r1  r   r   r   r   r   r   r5   r5   rz   r8   r'  &  s    



3r'  c                   @     e Zd ZdZdZdS )CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr/   r<   r=   r   r  r5   r5   r5   r8   rf        rf  c                   @  re  )
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNrh  r5   r5   r5   r8   rj    ri  rj  c                   @  re  )
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNrh  r5   r5   r5   r8   rl    ri  rl  c                   @  r   )_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    r   rT   rt   rs   r2   r   c                 C  sh   |  |s	tdS | ||stdS t|| }|j| |j|j| < |j	|j |j
|j
 |S )NZfunction_mismatchZmultiple_users)r$  r   r%  rl   rq   rg   rV  rr   r0   r   r1   rh   )r7   r   rt   r   r5   r5   r8   r     s   

z_TargetExprVarArgs._matchNr   r   r5   r5   r5   r8   rn    s    rn  c                   @     e Zd ZdZdS )CallFunctionVarArgsrg  Nr/   r<   r=   r  r5   r5   r5   r8   rp        rp  c                   @  ro  )CallMethodVarArgsrk  Nrq  r5   r5   r5   r8   rs    rr  rs  c                   @  ro  )CallModuleVarArgsrm  Nrq  r5   r5   r5   r8   rt  	  rr  rt  c                      sF   e Zd ZdZdd fd	d
ZdddZdddZd fddZ  ZS )ListOfz$
    Matches a repeated pattern
    Frn   rm   r   r   r2   rJ   c                   s(   t    t|tsJ || _|| _d S r4   )rx   rN   r   rm   rn   r   )r7   rn   r   rz   r5   r8   rN     s   

zListOf.__init__r.   c                 C     | j j d| j dS Nr  r   )r{   r/   rn   rM   r5   r5   r8   r     r   zListOf.__repr__r   rp   rt   rs   r   c           
      C  s   t |ttfrt|dkrtdS t|| }| }d}t|D ]1\}}t|j	||j
d}|| j|}	| }t|	sH| jsGtd||	  S q d}||	  q |sXtdS | S )Nr   Znon_listFr   zlist[{}]: {}Tzlist: no_match)r   ry   r   r   r   rl   r   	enumeraters   r   r~   r   rn   r   r   r   r   )
r7   r   rt   r   r   ZmatchedrR  rZ  Z	child_ctxr[  r5   r5   r8   r     s(   

zListOf._matchr   r   c                   s2   t t|}t |o| j|jo| j|jkS r4   )r  r  r   rx   r   rn   r   r   rz   r5   r8   r   3  s   
zListOf.pattern_eqF)rn   rm   r   r   r2   rJ   r   )r   rp   rt   rs   r2   r   r   r  r5   r5   rz   r8   ru    s    

ru  c                      sv   e Zd ZU ded< d$ fddZed%d	d
Zd&ddZd'ddZd(ddZ	d)ddZ
d*ddZd+ fd"d#Z  ZS ),MultiOutputPatternr   r   Sequence[Optional[PatternExpr]]r2   rJ   c                   sP   t    t|d tsJ tdd |D sJ |t|| _|d j| _d S )Nr   c                 s  s"    | ]}|d u pt |tV  qd S r4   )r   rm   r+  r5   r5   r8   ra   B  s     z.MultiOutputPattern.__init__.<locals>.<genexpr>)rx   rN   r   r	  rc  ry   r   r  )r7   r   rz   r5   r8   rN   ?  s
   

zMultiOutputPattern.__init__-Union[Callable[..., Any], str, Sequence[Any]]c                 C  s   t t| jd }|jS Nr   )r  r  r	  r   r
  )r7   outputr5   r5   r8   r
  F  s   zMultiOutputPattern.fnsr.   c                 C  rv  rw  )r{   r/   r   rM   r5   r5   r8   r   L  r   zMultiOutputPattern.__repr__r   r   c                   sD    fdd| j D }dd }| jj d|| }| d}|S )Nc                      g | ]}  |qS r5   rL  r+  rM  r5   r8   r   P      z3MultiOutputPattern.pretty_print.<locals>.<listcomp>z,
z  z([z
]))r   r{   r/   rK  )r7   r   r0   rN  Zstr_outr5   rM  r8   r   O  s
   

zMultiOutputPattern.pretty_printr   rT   rt   rs   r   c                 C  sr   t t| jd }|||}t|s|S | jdd  D ]}|d u r#q| ||}t|s1|  S || q|S )Nr   r*   )r  r  r	  r   r   r   _match_from_anchorsr   )r7   r   rt   r~  r   rn   r[  r5   r5   r8   r   V  s   zMultiOutputPattern._matchrn   rm   c                 C  sP   t |j}td}||t D ]}|||}t|r |  S t ||_q|S )Nzno anchor found)r   r   r   r   r    r   r   )r7   rn   rt   Zpriorr   r   r5   r5   r8   r  f  s   
z&MultiOutputPattern._match_from_anchorsc              
   C  sD   zt | j|jd| |W S  ty! } z|W  Y d }~S d }~ww r   )rs   r   r~   r   r   r   r5   r5   r8   r   s  r   zMultiOutputPattern.matchr   r   r   c                   sH   t t|}t |o#t| jt|jko#tdd t| j|jD S )Nc                 s  r]  r4   r^  r_  r5   r5   r8   ra   ~  rb  z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>)	r  r  r   rx   r   r   r   rc  rW  r   rz   r5   r8   r   y  s   zMultiOutputPattern.pattern_eq)r   r{  r2   rJ   )r2   r|  r   r   r   )rn   rm   rt   rs   r2   r   r   r   )r/   r<   r=   r>   rN   r   r
  r   r   r   r  r   r   r   r5   r5   rz   r8   rz  <  s   
 




rz  c                      sH   e Zd ZdZd fddZedd	d
ZdddZd fddZ  Z	S )RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    inner_patternr	  r2   rJ   c                   s   t    || _|j| _d S r4   )rx   rN   r  r  )r7   r  rz   r5   r8   rN     s   
zRepeatedExpr.__init__Sequence[FnsType]c                 C  r}   r4   )r  r
  rM   r5   r5   r8   r
    r   zRepeatedExpr.fnsr   rT   rt   rs   r   c                 C  sv   | | j|}t|s|S |j| j | j|t D ]}t| g|jd | j|}t|s3|  S |	| q|S r   )
r   r  r   r   popr   r    rs   r~   r   )r7   r   rt   r   Zanchor_nodeZanchor_mr5   r5   r8   r     s   zRepeatedExpr._matchr   r   r   c                   s&   t t|}t |o| j|jS r4   )r  r  r   rx   r   r  r   rz   r5   r8   r     s   zRepeatedExpr.pattern_eq)r  r	  r2   rJ   )r2   r  r   r   )
r/   r<   r=   r   rN   r   r
  r   r   r   r5   r5   rz   r8   r    s    
r  c                   @  sH   e Zd ZdZdddZeedddddZdddZ	dddZ
dS )r   z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    r2   rJ   c                 C  s   t jj | _i | _i | _d S r4   )rZ   r[   r~   Z
_Namespace	namespacememoized_objs_namesmemoized_objs_pprM   r5   r5   r8   rN     s   
zPatternPrettyPrinter.__init__Nr~  objrm   output_namer.   c                   sR   t   t| ds
J | j d} fdd jD }|| d|  d|S )zU
        Serializes obj to python code with obj written out to `output_name`
        r   rM  c                   s&   g | ]} j |  d  j|  qS ) = )r  r  )r^   r   rM  r5   r8   r     s    z,PatternPrettyPrinter.run.<locals>.<listcomp>r  
)r   hasattrr   r  rg   rK  )r  r  Zout_strr~  r5   rM  r8   run  s   

zPatternPrettyPrinter.runr   c                 C  sD   t |tr| j| }r|S | |S t|dr|| S t|S )Nr   )r   r'  r  re   memoizer  r   rJ  )r7   r  Zmemoized_namer5   r5   r8   r     s   



z!PatternPrettyPrinter.pretty_printr'  c                 C  sN   | | }| }dD ]}||d}q| j|d }|| j|< || j|< |S )N)zaten.r  zprims.rP   )r   r  replacer  create_namer  r  )r7   r  Zobj_strobj_nameprefixtmp_namer5   r5   r8   r    s   


zPatternPrettyPrinter.memoizerO   )r~  )r  rm   r  r.   r2   r.   )r  r   r2   r.   )r  r'  r2   r.   )r/   r<   r=   r   rN   rd  r   	lru_cacher  r   r  r5   r5   r5   r8   r     s    

r   c                   @  r?   )	_PassDictsTyper_    tuple[str, torch.fx.node.Target]r2   list[PatternEntry]c                 C  r3   r4   r5   )r7   r_   r5   r5   r8   __getitem__  rG   z_PassDictsType.__getitem__N)r_   r  r2   r  )r/   r<   r=   r  r5   r5   r5   r8   r    rB   r  c                   @  s8   e Zd ZU ded< ded< dddZ		ddddZdS )PatternEntryrm   rn   Callable[[Match], bool]extra_checkr   rl   r~   r|   r   rT   r2   rJ   c                 C  r  r4   r   r7   r   r~   r   r5   r5   r8   apply  r   zPatternEntry.applyNF
pass_dicts/Union[_PassDictsType, Sequence[_PassDictsType]]rV  !Union[torch.fx.node.Target, None]prependr   c                 C  s   |d u rt | jdsJ | jjD ]
}| j|||d qd S t|ttfrIt | jds,J |r<|| jj|f d|  d S || jj|f 	|  d S t
tt |}|D ]
}| j|||d qSd S )Nr
  r  r  r   )r  rn   r
  registerr   r   PatternMatcherPassr  insertrg   r  r  r
   r  )r7   r  rV  r  rD   r,  r5   r5   r8   r    s   zPatternEntry.registerr   rl   r~   r|   r   rT   r2   rJ   r   )r  r  rV  r  r  r   r2   rJ   )r/   r<   r=   r>   r  r  r5   r5   r5   r8   r    s   
 
r  c                   @  s    e Zd ZU ded< dddZdS )LoweringPatternEntryCallable[..., Any]handlerr   rl   r~   r|   r   rT   r2   rJ   c                 C  s   t | jt | j|}|| ||t|j|j}|j	
|j	 || W d    n1 s4w   Y  |jd |u sBJ |  d S )N)r   wrapsr  r   inserting_beforerg  r   r0   r1   ri   rh   replace_all_uses_withrq   r   )r7   r   r~   r   r  r   r5   r5   r8   r  
  s   zLoweringPatternEntry.applyNr  )r/   r<   r=   r>   r  r5   r5   r5   r8   r    s   
 r  c                   @  s$   e Zd ZU dZded< dddZdS )GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r  r  r   rl   r~   r|   r   rT   r2   rJ   c                 C  sL   | | | j|g|jR i |j W d    d S 1 sw   Y  d S r4   )r  r  r0   r1   r  r5   r5   r8   r    s   "zGraphPatternEntry.applyNr  )r/   r<   r=   r   r>   r  r5   r5   r5   r8   r    s   
 r  c                   @  s.   e Zd ZU ded< edddZdddZdS )r   zCallable[..., list[Any]]normalize_argsr   rl   r~   r|   ru   +Union[torch.fx.Graph, torch.fx.GraphModule]r0   Sequence[torch.fx.Node]r2   rJ   c                   s\  G fdddt jj}|  }t|dkr|d }n"|d s!J t|d jjfdd|D }t|t	
ddd }ddd|Y t|t jjsQJ ||j  }t|t jjrb|g}dddd fddt|t|krt||D ]	\}	}
|	|
 qnt|dksJ |d | W d    n1 sw   Y  |   d S )Nc                      s,   e Zd ZdZdZdZd fddZ  ZS )	z<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNr   rT   r2   r   c                   s   |j dv rt |S |j dkr[|j}| |\}}t|s J |||}t|j|dd d|jv rYd|jvrY|jd |jd< t	|jd t
jrYd|jv sQJ |jd |jd< |S td| )N)placeholderr~  rg  ZInterpreter_Replacerr   r   Ztensor_metaz
unhandled )r  rx   run_noderV  Zfetch_args_kwargs_from_envr  rg  rk   ri   r   rZ   Tensorr!  )r7   r   rV  r0   r1   result)r{   r~   r5   r8   r  1  s&   

zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_noder   rT   r2   r   )r/   r<   r=   rk  rm  get_attrr  r   r5   r   rz   r8   Replacer,  s
    r  r*   r   c                   s(   g | ]}t |tjjr ||fqS r5   )r   rZ   r[   r"  index)r^   r   )rq   r5   r8   r   M  s    z>ReplacementPatternEntry.replace_with_graph.<locals>.<listcomp>)r   r   rT   tag_namer.   	tag_valueinput_stopsr   r2   rJ   c                 S  sf   | g}t tjj  }|r1| }||vr-||vr-t|dr-|| ||j|< ||j	 |sd S d S )Nri   )
r    rZ   r[   r"  r  r  r\  ri   r   Zall_input_nodes)r   r  r  r  queuevisitedr   r5   r5   r8   percolate_tagsT  s   

zBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tagsr   c                 S  s:   | j dkrd S | jtjkrd S t| jdksJ | jd S )Nrg  r!   r*   )r  rV  r  getitemr   r0   r   r5   r5   r8   maybe_getitemn  s   

zAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitemoldUnion[torch.fx.Node, None]new3Union[torch.fx.Node, Sequence[torch.fx.Node], None]c                   s   | d u r|d u s
J d S t | tjjsJ |d u r%| d  |  d S t |tjjrZd|jvr8|j| j dD ]}|| jv rM||| j| t  q:| | |  d S t	| j
 }|D ]}|}|d u rqtd|||  qc|  d S )Nr   )Z	recomputeZac_graph_idzcan't handle)r   rZ   r[   r"  r  r   ri   rh   r    ry   r   r   AssertionError)r  r  r  Zold_usesuseridx)r0   r~   r  r  r  r5   r8   r  v  s6   






z;ReplacementPatternEntry.replace_with_graph.<locals>.replace)
r   rT   r  r.   r  r.   r  r   r2   rJ   r  )r  r  r  r  r2   rJ   )rZ   r[   Interpreterr   r   ry   r~   rq   minr  
itemgetterr  r   GraphModuler  r"  rW  r   )r   r~   ru   r0   r  r   Z	last_nodeindicesr   r  r  r5   )r0   r~   r  rq   r  r  r8   r   %  s6   



EZz*ReplacementPatternEntry.replace_with_graphr   rT   c              	   C  s4   |j d usJ | |||j | j|ji |j d S r4   )ru   r   r  r0   r1   r  r5   r5   r8   r    s   zReplacementPatternEntry.applyN)
r   rl   r~   r|   ru   r  r0   r  r2   rJ   r  )r/   r<   r=   r>   rd  r   r  r5   r5   r5   r8   r   !  s   
  r   r   r   c                 C  r   r   r5   r   r5   r5   r8   _return_true  r   r  	search_fnr  r   r   c                 C  s   t d| j| d S )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor/   )r  r   r5   r5   r8   log_trace_failure  s
   r  Frn   r~   Optional[torch.fx.Graph]seen_patternsdict[str, list[Optional[str]]]skip_duplicatesc                   s   t | |}|s| |rt| dS d dS |du r0|r&dS tdfdd t|}|D ] | ks=q6|rB dS td fdd q6|| dS )aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     s   d  dS )NDuplicate pattern: z with no graphr5   r5   )pattern_reprr5   r8   r     r>  z1check_and_add_duplicate_pattern.<locals>.<lambda>c                     s   d d  dS )Nr  z with duplicated match graph  r5   r5   Z	graph_strr  r5   r8   r     s    )r   r  re   rg   r.   rZ   _check)rn   r~   r  r  Zequiv_pattern_reprsZnew_graph_strr5   r  r8   check_and_add_duplicate_pattern  s4   



r  r5   
replace_fnexample_inputsIterable[Any]r   r  r  r  r  scalar_workaround)Union[dict[str, Union[float, int]], None]exclusive_arg_namesSequence[str]search_fn_patternUnion[PatternExpr, None]c
              
     s(  g t j  d f	dd}
d fd
d}tu r*t r*dS tjddZ dd |D du rGt	|\}}n}d}t
|trR|n|gD ]}t
|trst||rc|jnd|j|	drs W d   dS qUt||
|d}|| |jW  d   S 1 sw   Y  dS )a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    r   rl   r2   r   c                   sj  t }|D ]}|jvrtd| dj qt tjfdd|D dd  g }tjj u t	D ]^\}}t
 | tjr|rXt | jrX W d   dS tj |   |   | j | j|d	 |< t | j |  D ]t
tjrtfd
d|D r| qq:
}|sZ|r+d 	fdd}z	||  }W n ty }	 zt	|	 W Y d}	~	W d   dS d}	~	ww g }
ttt|t  |jjD ]C\}}|t|k r|
|j q|j|# |j ||t|  }|j!|_|"| |j#| W d   n	1 s w   Y  q|
| }n'z	 }W n tyQ }	 zt	|	 W Y d}	~	W d   dS d}	~	ww t$||d}% d }|dusgJ |&|}t'|r|r _(tjdkrj(jjD ]}t)|j*jd dd q	 W d   dS 	 W d   dS 1 sw   Y  dS )z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                   s   g | ]} j | qS r5   r   r^   r   r  r5   r8   r   0  r  z:register_replacement.<locals>.check_fn.<locals>.<listcomp>c                 S  r   r   r   )r   r5   r5   r8   r   0  r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>NF)dtypedevicerequires_gradc                 3  s    | ]	}t  |kV  qd S r4   r   r^   r`  )r`   r5   r8   ra   C  s    
z9register_replacement.<locals>.check_fn.<locals>.<genexpr>args_newr   r2   c                    s   | t | t   d   S r4   )r   )r  )r0   r  r5   r8   search_fn_newV  s   z=register_replacement.<locals>.check_fn.<locals>.search_fn_new)argnamesr  r  r   r*   r   r   T)r  r   r2   r   )+ry   r1   r   rZ   r[   r   Z_dynamoutilsZdetect_fake_moderx  r   r  r   r  Zempty_stridedsizeZstrider  r.  r/  shapeZSymIntrc  rg   r  rW  ranger   r~   rq   rV  Zinserting_afterr  r   r  r   fx_to_patternr   r   r   ru   rk   ri   )r   r  r   Zsym_argsrR  ZgradZspecific_patternr  Zspecific_graphr   Zsym_arg_namesr  new_noder   Zspecific_pattern_matchr   	argnames_staticr  r  r  r  r  r  r  r   )r0   r   r`   r8   check_fn  s   






&



@

Z&z&register_replacement.<locals>.check_fnr1   r   ro   c                    sh    fddD }t dt d D ]}d|  vr n| d|  q r2J d |S )Nc                   r  r5   )r  r  r   r5   r8   r     r  z@register_replacement.<locals>.normalize_args.<locals>.<listcomp>r*   Z	tangents_zleftover kwargs: )r  r   rg   r  )r1   r0   rR  )r  r   r8   r    s   z,register_replacement.<locals>.normalize_argsFZfunctionalize_rng_opsc                 S  s   g | ]}t |tjo|jqS r5   )r   rZ   r  r  r+  r5   r5   r8   r     s    z(register_replacement.<locals>.<listcomp>N)r  )rn   r  r  r   rl   r2   r   )r1   r   r2   ro   )inspect	signature
parametersr   joint_fwd_bwdrZ   Zis_inference_mode_enabledfunctorch_configpatchgen_pattern_and_search_gmr   r
   r  r  r~   r  r   r  rn   )r  r  r  r   r  r  r  r  r  r  r  r  rn   gmZpattern_matcher_passr5   r  r8   register_replacement  sP   r	


$r  unique_namer   c                 C  s  ddd}t  stdt  |j}ddlm} |jdd	 t||||}W d    n1 s1w   Y  tj	|| d
}	|t
vrId}
t
| nd}
| }tt | d |
"}|
dkrc|| n|d ||	 |d W d    |S 1 s}w   Y  |S )Nr2   r.   c                  S  s   t d} t dj| d}g }ttjjD ]}ttjj|}t|t	r0t
|ttfr0|| qd|}d| d}| | S )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirrZ   Z	_inductorpattern_matcherr  r   r:  
issubclassrm   r	  rg   rK  )Zauto_generated_msgfile_templateZpattern_matcher_importsr   attrZformatted_importsr5   r5   r8   get_file_template  s"   

z-_serialize_pattern.<locals>.get_file_templatez0Could not find serialized patterns directory at r   r"   Fr  )r  wr`  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r/   Ztorch._functorchr#   r  gen_patternr   r  _serialized_patternsr\  openwrite)r  r  r  r   r  r  pattern_namer   rn   Zserialized_patternZ
write_moder  fr5   r5   r8   _serialize_pattern  s4   
#


r  Z	fx_passesZserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                 C  s   t |}dtjv rt| ||||}
n|j}td| }|r$t|| s*t	d|  t
|| }
t|D ]}t|trC|jd urCd |_q4t|||||
f t|||||||||
|	d
 d S )NZPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  r  )r   osenvironr  r/   	importlibimport_moduler  r  warningr  r@  Z	tree_iterr   r'   Zconstantr  rg   r  )r  r  r  r  r   r  r  r  r  r  patr  r   r   r5   r5   r8   gen_register_replacement#  sD   



r   r  (tuple[PatternExpr, torch.fx.GraphModule]c           
      C  s   g t | j }|d u ri }g }d}|D ]}||v r$|||  q|||  |d7 }q|| |}	t|	tttt	j
t	jf|||d|	fS )Nr   r*   )ignore_typesr  r  r  )r  r  r  r   rg   r  intfloatry   rZ   r  r  )
r  r  r   r  r  r  Zflat_inputsZ	input_idxargnameZ	search_gmr5   r5   r8   r  [  s(   

r  c                 C  s   t | ||||d S r}  )r  )r  r  r   r  r  r5   r5   r8   r  ~  s
   
r  r  	pass_dictr  2Callable[[Callable[..., Any]], Callable[..., Any]]c                     d fdd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    r  r  r2   c                   s.   t | sJ t | djd d| _| S )Nrn   r  r  r  T)r  r  r  Z_inductor_lowering_functionr  r  r&  rn   r  r5   r8   	decorator  s   z,register_lowering_pattern.<locals>.decoratorNr  r  r2   r  r5   rn   r  r&  r  r,  r5   r+  r8   register_lowering_pattern  s   r/  c                  r(  )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    r  r  r2   c                   s(   t | sJ t | djd | S )Nr)  r  )r  r  r  r*  r+  r5   r8   r,    s   z)register_graph_pattern.<locals>.decoratorNr-  r5   r.  r5   r+  r8   register_graph_pattern  s   r0  r|   r   c                 C  s   |t t| ju S r4   )r   iterrq   )r~   r   r5   r5   r8   is_start_of_fx_graph  s   r2  z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)r  torch._ops.OpOverloadc                 C  s*   | j dkrdS | tjjjjtjjjjfv S )NinductorF)r  rZ   opsr4  Zaccumulate_grad_defaultZresize_storage_bytes_)r  r5   r5   r8   "fixme_incorrect_inductor_schema_op  s   


r7  c                 C  s   t | jtjjrt| js| jjjS t | jtjj	j
rdS | jdkr3t| js)J t| jjr2dS n| jdkrHt | jts@J t| jrHdS | jdd uS )NFrg  Trk  out)r   rV  rZ   r  r  r7  Z_schemaZ
is_mutableZ_higher_order_opsZauto_functionalizeZAutoFunctionalizedr  r  _mutation_op_researchr/   r.   r1   re   r  r5   r5   r8   is_mutation_op  s(   



r;  r`  ra  c                 C  s0   d| j v sJ d|j v sJ | j d |j d kS Nmutation_region_idr   )r`  ra  r5   r5   r8   same_mutation_regions  s   r>  r#  c                 C  st   |}d|j vrt| |s|j}d|j vrt| |r|j dd}||ur8|j}t|r/|d7 }||j d< ||us$|S )Nr=  r   r*   )ri   r2  prevre   r   r;  )r~   r   r   r=  r5   r5   r8   get_mutation_region_id  s   
r@  c                 C  s   dt t| jjvS r<  )r   r1  rq   ri   r   r5   r5   r8   "should_compute_mutation_region_ids  r   rA  c                 C  s.   d}| j D ]}t|r|d7 }||jd< qd S )Nr   r*   r=  )rq   r;  ri   )r~   r=  ndr5   r5   r8   compute_mutation_region_ids  s   
rC  c                      s@   e Zd Z	dd fddZdddZdddZdddZ  ZS )r  NrU   Optional[str]r2   rJ   c                   s(   t    tt| _|| _tt| _d S r4   )rx   rN   r   ry   patternsrU   r  )r7   rU   rz   r5   r8   rN     s
   
zPatternMatcherPass.__init__itemr  r  c                 C  s
   | j | S r4   )rE  )r7   rF  r5   r5   r8   r    r   zPatternMatcherPass.__getitem__r  +Union[torch.fx.GraphModule, torch.fx.Graph]r#  c              
   C  s4  | j sdS t|tjjr|j}nt|tjjr|}|j}n	tdt	| t
|r.t| tt|}d}g }d}| j D ]\}}|dkrHd}q=||j||dd q=|r`||jddd | jd urh| jnd}	t|tjjssJ t||	 ttj|dd	D ]}
t|
}|
jdkr|
j|f| j vrqt|
dd
rq| j |
j|f D ]_}|
jr nY|j|
}t|rttt ||j!dkrqt"j#$d|
j%krt&'d|
|
j(||j t|r|)|r|d7 }|*|||
 t+d d  d7  < t+d d  t|j!7  < qqW d    |S 1 sw   Y  |S )Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Frm  T)r  rV  sort)r  rH  r
  )reverse)Zallow_cpu_inputsr*   Z!TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sr4  Zpattern_matcher_countZpattern_matcher_nodes),rE  r   rZ   r[   r  r~   ZGraphowning_moduler   r:  rA  rC  r   r   r@  rg   Z
find_nodesrU   r   sortedr.  r/  from_iterabler#  r  r,   r   rn   r   r   r   r    rI  rq   r  r  re   r   r  r  r0   r  r  r   )r7   r  r~   Zget_mutation_region_id_partialrX  rq   Zhas_call_moduler  rV  rU   r   entryr   r5   r5   r8   r    sv   

!!zPatternMatcherPass.applyc                 C  s   | j   d S r4   )rE  clearrM   r5   r5   r8   rN  U  r   zPatternMatcherPass.clearr4   )rU   rD  r2   rJ   )rF  r  r2   r  )r  rG  r2   r#  rO   )r/   r<   r=   rN   r  r  rN  r   r5   r5   rz   r8   r    s    

@r  r0   r   r1   r   c                  O  r  r4   r   r   r5   r5   r8   _not_implementedY  r   rO  r  rG  r"  Sequence[type[Any]]r  c                   s   |pi }dd |  D tt|ksJ 	ddfd
dt G  fdddtjj}t| tjjs>J || 	 }t|t
sPtt|S |S )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                 S  s   i | ]\}}||qS r5   r5   r]   r5   r5   r8   r   k  r  z!fx_to_pattern.<locals>.<dictcomp>Nr,  rH   ignore_types_overrideOptional[Sequence[type[Any]]]r2   Union[T, KeywordArg, Ignored]c                   sn   |d ur|n }t | ttfr| v rt|  S t| |v r"t S t | tr5tdd | D r5| r5t S | S )Nc                 s  s    | ]}t |tV  qd S r4   )r   r   )r^   yr5   r5   r8   ra   x  r  z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>)r   r$  r#  r   r:  r   ry   rc  )r,  rQ  Zcurrent_ignore_types)r"  inv_scalar_workaroundr5   r8   process_argn  s    z"fx_to_pattern.<locals>.process_argc                      sL   e Zd ZeZeZeZdfd	d
ZdfddZd fddZ	  Z
S )z fx_to_pattern.<locals>.ConverterrV  r.   r0   r   r1   Mapping[str, Any]r2   &Union[ExclusiveKeywordArg, KeywordArg]c                   s`   t }|t k r | }n r|dsJ |}n	tdd|}|}|v r,t|S t|S )NZtangentz_\d+$rP   )r   r   
startswithresubr  r   )r7   rV  r0   r1   r   r   )r  argnumr  r5   r8   r    s   
z,fx_to_pattern.<locals>.Converter.placeholderrm   c                   s    |t jkrtdd D fdfd	d
}| t ||f\}}tv r; fdd|D } fdd| D }t|g|R i |S )Nc                 s  s    | ]	}|t ur|V  qd S r4   )r#  )r^   tr5   r5   r8   ra     s    zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>r,  rH   rQ  rR  r2   rS  c                   s
    | |S r4   r5   )r,  rQ  )rV  r5   r8   process_arg_fn_impl  s   
zKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_implc                   s   g | ]} |qS r5   r5   r  Zprocess_arg_fnr5   r8   r     s    zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<listcomp>c                   s   i | ]	\}}| |qS r5   r5   )r^   r_   r`  r_  r5   r8   r         zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<dictcomp>r,  rH   rQ  rR  r2   rS  )r  r  r   r@  rA  ry   rj   rf  )r7   rV  r0   r1   r^  )r"  rV  r_  r8   rg    s   
z.fx_to_pattern.<locals>.Converter.call_functionr   rT   r   c                   s   t  |}|jdkr8t|tr8|jd }t|tsJ t|t|ks&J t||D ]
\}}t|j	|_	q+|S t|j	|_	|S )Nr~  r   )
rx   r  r  r   r   r0   r   r   rW  r   )r7   r   rvr0   rr   rz   r5   r8   r    s   
z)fx_to_pattern.<locals>.Converter.run_node)rV  r.   r0   r   r1   rW  r2   rX  )rV  r.   r0   r   r1   rW  r2   rm   )r   rT   r2   r   )r/   r<   r=   rO  rk  rm  r  r  rg  r  r   r5   )r  r\  r  r"  rV  rz   r8   	Converter~  s    rd  r4   ra  )rj   r   r.  rX  rZ   r[   r  r   r  r  rm   rz  r@  Ztree_leaves)r  r"  r  r  r  rd  rn   r5   )r  r\  r  r"  rU  rV  r8   r  ]  s    @
r  T)r   get_decomp_fnrD   r   re  Optional[Callable[..., Any]]rF   c                C  s|   t   |dur| nt }t| |dd| }W d   n1 s!w   Y  ddlm} |r8||j |j  |  |S )z>Build a normalized inference graph, for use with fx_to_patternNreal)Ztracing_moder*   remove_noop_ops)r   r+   r   fx_passes.post_gradri  r~   eliminate_dead_code	recompile)rD   r0   r   re  decompositionsr  ri  r5   r5   r8   r     s   


r   c              	     s   d d fd
d}t jd t| dd |t ddd|  W d   n1 s)w   Y   s2J ddlm} | j ddlm	} t
 }tt jjjjtdtd}t||td|j | j t jj  j_ j      S )z=Build a normalized training graph, for use with fx_to_patternNjoint_graphrF   inputsr   r1   r   r2   1tuple[torch.fx.GraphModule, torch.fx.GraphModule]c                   s"    rJ t |  t| |fi |S r4   )clone_graphr&   )rn  ro  r1   r  r5   r8   record_joint_graph  s   z)joint_fwd_bwd.<locals>.record_joint_graphc                 S  r   r4   )r%   )grR  r5   r5   r8   r     s    zjoint_fwd_bwd.<locals>.<lambda>TF)Zpartition_fnrm  Zkeep_inference_input_mutationsZ
enable_logr*   rh  )pointless_viewr   r  )rn   r  r  )rn  rF   ro  r   r1   r   r2   rp  )rZ   _guardstracingr$   r+   rj  ri  r~   Zfx_passes.joint_graphru  r  rf  r5  atenviewr6  r   r  r  r  rE  r  r[   ZCodeGenZ_codegenrk  rl  )rD   r0   rs  ri  ru  Zmatcher_passrn   r5   rr  r8   r    s>   	


r  r   list[torch.fx.node.Argument]c                 C  s    g }t j| j| jf|j |S r4   )rZ   r[   r   r0   r1   rg   )r   r0   r5   r5   r8   rY    s   rY  c                   s   t t| j}ttjj   tt }d }|rQ| } fddt	|D }|r1||d  
| n | |rB|j|urB|
| |}|t||d |s|s\t t| jks^J d S )Nc                   s   g | ]}| vr|qS r5   r5   r+  readyr5   r8   r   '  s    z+stable_topological_sort.<locals>.<listcomp>r  r5   )ry   r   rq   r    rZ   r[   r"  r   r  rY  rg   r\  r   r   r   )r~   pendingZwaitingcursorr   Zwaiting_forr5   r{  r8   stable_topological_sort  s    

r  Callable[[], Any]c                   s&   t dt  d fdd}|S )z0Wrapper around lazy init functions in fx_passes/Nr2   r   c               
     s   t d  } tjd 1 t  t    }W d    n1 s"w   Y  W d    n1 s1w   Y  W d    n1 s@w   Y  | t d< |S )Nr4  )r   rf   rZ   rv  rw  r   r(   )Zcounters_refr  r  r5   r8   	lazy_init;  s     z%init_once_fakemode.<locals>.lazy_init)r2   r   )r   r  r  )rD   r  r5   r  r8   init_once_fakemode8  s   r  r   Callable[[Match], Any]c                   s   d fdd}|S )	z2Function for extra_check to put pass behind a flagr   rl   r2   r   c                   s
   t t S r4   )r  r#   r  r   r5   r8   
flag_checkN  r   zconfig_flag.<locals>.flag_checkN)r   rl   r2   r   r5   )r   r  r5   r   r8   config_flagK  s   r  input_graphc                 C  s   G dd dt }||  S )Nc                      s   e Zd Zd fddZ  ZS )zclone_graph.<locals>.CopyGraphrS   rT   r2   c                   sD   t  |}t|tjjr |jj|j | j	j
|jd |j_|S r4   )rx   r  r   rZ   r[   ZProxyr   ri   rh   Z	new_graphZ_graph_namespacer  r   )r7   rS   r  rz   r5   r8   r  V  s   z'clone_graph.<locals>.CopyGraph.run_node)rS   rT   r2   rT   )r/   r<   r=   r  r   r5   r5   rz   r8   	CopyGraphU  s    r  )r)   Z	transform)r  r  r5   r5   r8   rq  T  s   
rq  
arg_number
kwarg_namerD  c                 C  s0   t | j|kr| j| S |d u rd S | j|S r4   )r   r0   r1   re   )r   r  r  r5   r5   r8   get_arg_valuef  s
   
r  rq   Iterable[torch.fx.Node]rp   c                   sB    gt  tjjr fdd  D  fdd| D S )Nc                   s   g | ]}t  |qS r5   r  r  r  r5   r8   r   t  r  z filter_nodes.<locals>.<listcomp>c                   s   g | ]	}|j  v r|qS r5   )rV  )r^   r   )r
  r5   r8   r   v  r`  )r   rZ   r  r  r   r  )rq   rD   r5   )rD   r
  r8   filter_nodesq  s   r  torch.fx.node.Targetc                 C  s2   | j dkrt| jtsJ t| jj| jjS | jS )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    rm  )r  r   rV  r.   r   r~   rJ  r{   r  r5   r5   r8   r#  y  s   
r#  )rP   )rQ   rR   rS   rT   rU   r.   r2   rJ   )r   r   r2   r   r  )r  r  r   r   r2   rJ   ry  )
rn   rm   r~   r  r  r  r  r   r2   r   )r  r-   r  r@   r  r  r   rC   r  r  r  r  r  r  r  r  r  r  r  r   r2   r   )r  r.   r  r-   r  r   r   rC   r  r  r2   rm   )r  r.   r  r-   r  r@   r  r  r   rC   r  r  r  r  r  r  r  r  r  r   r2   rJ   )Nr5   )r  r-   r  r   r   rC   r  r  r  r  r2   r!  )r  r-   r  r   r   rC   r  r  r  r  r2   rm   )
rn   rm   r  r  r&  r  r  r   r2   r'  )r~   r|   r   rT   r2   r   )r  r3  r2   r   r&  )r`  rT   ra  rT   r2   r   )r~   r|   r   rT   r2   r#  )r~   r|   r2   r   )r~   r|   r2   rJ   )r0   r   r1   r   r2   r   )r5   r5   Nr5   )r  rG  r"  rP  r  r  r  r  r  r  r2   rm   )
rD   r  r0   r   r   r   re  rf  r2   rF   )rD   r  r0   r   r2   rF   )r   rT   r2   rz  )rD   r  r2   r  )r   r.   r2   r  )r  rF   r2   rF   r4   )r   rT   r  r#  r  rD  r2   r   )rq   r  rD   r   r2   rp   )r   rT   r2   r  )r   
__future__r   r   dataclassesr   r  r  r.  loggingr  r  rZ  r  r  abcr   r   collectionsr   collections.abcr   r   r   r	   r
   pathlibr   r   r   r   r   r   r   r   Ztyping_extensionsr   r   rZ   Ztorch._guardsZtorch.fxZtorch.utils._pytreer  Z_pytreer@  Ztorch._dispatch.pythonr   Ztorch._dynamo.utilsr   Ztorch._prims_commonr   Ztorch._subclasses.fake_tensorr   Z"torch.fx.experimental.proxy_tensorr   Z%torch.fx.experimental.symbolic_shapesr   Ztorch.fx.graph_moduler   Ztorch.fx.immutable_collectionsr   r   Z(torch.fx.passes.graph_transform_observerr   Ztorch.utils._ordered_setr    Z
_functorchr#   r   Z_functorch.aot_autogradr$   r%   Z_functorch.partitionersr&   Z_subclassesr'   r(   r[   r)   rP   decompositionr+   Zloweringr,   	getLoggerr/   r  r5  rx  ZprimsConstantr"  r   r-   r@   rC   rH   r   ZTargetr.   ZFnsTyperI   rK   rk   rl   r   r   r   r   rs   rm   r   r   r   r  r	  r   Z_SimpleSpecr'  rf  rj  rl  rn  rp  rs  rt  ru  rz  r  r   r  	dataclassr  r  r  r   r  r  r  r  r  r  __file__parentr  r  r>   r   r  r  r  r/  r0  r2  compiler9  r7  r;  r>  r@  rA  rC  r  rO  r  Zno_gradr   Zenable_gradr  rY  r  r  r  rq  Z_seen_patternsr  r  r#  r5   r5   r5   r8   <module>   s>   #$
 
*#
R 0/I&7 
-
0 
C
J

8&
Xh.$	
