o Zh@sddlmZmZmZmZddlZddlmZddlm Z ddl m Z ddl m Z mZddlmZmZdd lmZmZmZdd lmZmZmZmZmZmZmZmZmZm Z d ed e d ee!fddZ"d ed e dede!de!de!de!de!de!de#de#dee!d efddZ$d e de%ee&e!e!ffde%ee&e!e!ffdede!d e f ddZ'd ed!ed"e d#ed$eej(e)fd%eej(e#fd&e!d efd'd(Z*d!ed)ed eee+efd*e d"e d#ed+e!ded,e%e!e,efd eee+effd-d.Z-d!ed*e d"e d#ed ef d/d0Z.d1ed*e d2e#d e/fd3d4Z0d5eee+efd6eeee+efd1ed*e d"e d+e!d efd7d8Z1d5eee+efd6eeee+efd!ed*e d"e d+e!d efd9d:Z2 dAd;e!d*e de/d,ee%e!e,efd e fd?d@Z3dS)B)AnyCallableOptionalUnionN)get_node_type_to_io_type_map)get_new_attr_name_with_prefix)_is_activation_post_process) GraphModulemap_arg)GraphNode)NSNodeTargetTypeNSSingleResultValuesType NSSubgraph) get_arg_indices_of_inputs_to_log$get_node_first_input_and_output_typeget_node_input_qparamsget_normalized_nth_inputget_number_of_non_param_argsget_target_type_strgetattr_from_fqnNodeInputOrOutputTypeop_type_supports_shadowingreturn_first_non_observer_nodenodegmreturncCs`d}t|dr.|}|jdkr&t|jtsJt||j}t|r&t||d}|j|j d}|S)N_node_name_to_scope call_moduler) hasattrop isinstancetargetstrrrrrname)rrfqnZnode_to_use_for_fqnmoduler(J/var/www/auris/lib/python3.10/site-packages/torch/ao/ns/fx/graph_passes.py_maybe_get_fqns    r* logger_clslogger_node_name_suffix ref_node_name model_nameref_nameref_node_target_type results_typeindex_within_arg index_of_argr&c  Cs\t|j||} t||} |||j||| ||| | | }t|| ||jd| |fi}|S)z Given a starting graph of prev_node -> node -> next_node This function creates a new logger_cls obj and adds it after node, resulting in prev_node -> node -> logger_obj -> next_node r)rr%rsetattrgraph create_node)rrr+r,r-r.r/r0r1r2r3r&Zlogger_node_nameZ target_typeZ logger_objZ logger_noder(r(r)_insert_logger_after_node*s*  r7*node_to_instrument_inputs_to_ref_node_name+node_to_instrument_outputs_to_ref_node_namecst}ifdd}|jjD]}|jdkr#|tt||d|q||vs+||vrt||}||vr||\} } t|} | D]Y} t||| } t | t krj| j }t |||d|j || | t jjd| |d | j <q@t | tjjjkrt| D]\}}|j }t |||d|j || | t jj|| |d |j <qwq@ q@||||j <||vr||\} } t |j ||d|j || | t jjdd|d |j <q||||j <qt||}|S)z Takes the graph of gm, adds loggers to the output of each node in nodes_to_instrument. Returns a GraphModule with the new graph. ct|fddS)Nc |jSNr%renvr(r)k z8add_loggers_to_model..load_arg..r ar?r(r)load_argjz&add_loggers_to_model..load_argoutputrZ _ns_logger_r2r3r&)r r5nodesr!rHr rr*rtyper r%r7r NODE_INPUTvaluetorchZfxZimmutable_collectionsZimmutable_list enumerate node_copy NODE_OUTPUTr )rr8r9r+r.Z new_graphrFrr&r/Z ref_node_typeZarg_indices_to_logZ node_arg_idxZnode_argZ prev_nodearg_idxargZnew_gmr(r?r)add_loggers_to_modelZs           rT prev_node_cnode_agm_bgraph_cscale zero_pointdtype_cast_namec Cs~t|jd|}t||||d|di|}t|jd|} t|| ||d| di| } |dtj||| tjfi|S)NZ _input_scale_get_attrr(Z_input_zero_point_ call_function)rr%r4r6rNquantize_per_tensorZquint8) rUrVrWrXrYrZr[Zscale_node_nameZ scale_nodeZzero_point_node_nameZzero_point_noder(r(r) _insert_quantize_per_tensor_nodes*     r_node_cgm_anode_name_prefixnode_type_to_io_type_mapc CsZd} d} d} d} d} d}t||||\}}t||||\}}|tjkr(|tjks<|tjkr2|tjks<|tjkr@|tjkr@tj} nQ||krN|tjkrNtj j } nC|tjkrj|tjkrjt |||}|duritj } |\} }n'|tjkrz|tjkrzd} tj } ntd|d|d|d|dt|trt||}| r| dur|durt||||| ||S|d| |fi|S| r|d| || fi|S| sJ| }t||||d ||fi|St|tr#g}|D]8}t||}| r|d| |fi|}||q| sJ| }t||||d ||fi|}||q|Std t|d ) a Given a starting graph C (derived from graph B) of ... -> prev_node_c -> node_c -> ... And a corresponding related node_a, inserts the correct dtype cast node after prev_node_c to cast into the dtype expected by node_a, resulting in: dtype_cast / ... -> prev_node_c -> node_c -> ... For example, if node_c is an int8 op and node_a is an fp32 op, this function will insert a dequant. Ntozdtype cast from  z to z needs to be implementedr] call_methodrztype fz is not handled)rrFP32INT8ZFP16Z FP32_OR_INT8rN dequantizeUNKNOWNnnZIdentityrr^Zfloat16AssertionError format_noder"r rr_r6r4listappendrK)rVr`rUrarWrXrbr+rcZ dtype_cast_opZdtype_cast_mod_clsZdtype_cast_methodZdtype_cast_method_dtypeZdtype_cast_scaleZdtype_cast_zero_pointnode_input_type_aZ_node_output_type_aZnode_input_type_cZ_node_output_type_cnode_a_input_qparamsZnew_dtype_cast_nameZdtype_cast_modresultsZprev_node_c_innerZnew_dtype_cast_noder(r(r)_insert_dtype_cast_after_nodes                      rsc Cs<|jdkr/t|jd|}t||j}t|r|}t|||| |j|di|}|S|jdkr|jdvsBJd|jd|jdkrit t ||d |||}t|jd|}| |j|j|fi|}|St t ||d |||}t|jd|}| |j|j|t ||d fi|}|St d | d |jd) z+ Simple copy of node_a to graph_c. r\ _shadow_copy_r(rfrirdztarget  is not implementedrirr zhandling of node z with op )r!rr%rr#rNZ is_tensordetachr4r6_copy_node_from_a_to_crrlrm)rVrarWrXZnode_a_copy_nameZ node_a_objZ node_a_copyZarg_copyr(r(r)rxs`         rx subgraph_anum_non_param_args_node_ac Cs6g}|j}||jkr||t||d}||jks |||dd}|D]o}||dur3|nd}|j|dd}|durE|\} } n|j|j} } d} | t| krv| dkrYn| dkrb|dkrbn || | |sld S| d7} | t| ksT| D]} | dkrn| dkr|dkrn || |sd S| d7} qzq)dS) z This function returns `False` if the input subgraph cannot be copied by `_insert_copy_of_subgraph_a_after_input_node_c`. This usually means that there is a corner case logic for which copy is not yet implemented. rcSsft|trt||}|jdkr|jdvS|jdkrdSdSt|ttfr1|D] }t|ts0dSq&dS)Nrfrur\TF)r"r rr!r#rntuple)Z node_a_argraZarg_aelr(r(r) _can_inserts      z3_can_insert_copy_of_subgraph_a.._can_insertr TZnormalize_to_only_use_kwargsNF) end_node start_noderorreversenormalized_argumentsargskwargslenvalues) ryrarzrJcur_noder}rVZlocal_num_non_param_args_node_anorm_args_kwargs norm_args norm_kwargscur_idx kwarg_valr(r(r)_can_insert_copy_of_subgraph_asL           r input_node_cinput_node_c_2c Cst|ttfs J|jg}|j}||jkr&t||d}|d|||jks|d}t||||||} tdt |D]} || }| } t| d||||} q:| S)z* TODO(before land): real docblock rr N) r"r rnrrrinsert)_insert_copy_of_node_a_after_input_node_cranger) rrryrarWrbZ nodes_of_arZ cur_node_aZ cur_node_cZ cur_idx_arUr(r(r)-_insert_copy_of_subgraph_a_after_input_node_cs.       rcst|tr |jn t|tsJ|dj|jdd}|dur%|\}}n|j|j}}g} i} fdd} d} | t|krh| dkrH|} n| dkrS|durS|} n| || } | | | d7} | t|ksA| D]$\}| dkry|| |<n| dkr|dur|| |<n| | |<| d7} qlt | } t |}|j dkrt |}t|j tsJt|j }t|||j || | |}|S|j d vsJ|j |j | | |}|S) a Assume that node_a from graph_a has args (input, (input2)?, arg1, ...), and kwargs {kw0: kwarg0, ...} Note: input2 is optional. If it equals to None, we assume that the op has a single non-param input. If it is specified, we assume that the op has two non-param inputs. Copies the underlying values of arg1..argn and kwarg0..kwargn into gm_b, and creates the corresponding nodes in graph_c. Note: observers are ignored, so if an arg is an observer we navigate up until we find a non-observer parent. If node_a is a call_module, points the module pointed to by node_a to gm_b. Creates the copy of node_a in graph_c, with input as the first arg, and all other args and kwargs pointing to the copies of the objects in gm_b created above. An example in pictures: graph A: ======== input -------------> node_a / / / (input_2)?----------/ / / / / weight -> weight_obs / / bias ---------------- graph C (derived from B): ========================= input_node_c --> node_a_copy / / / (input_node_c_2)? / / / / weight_copy ----/ / / bias_copy ------/ rTr~Ncs~t|trt|}t|}|St|tttjfr|Sttt fr5D] }t|tr2Jdq'|St dt d)Nz/handling of Node inside list is not implementedzhandling for kwarg of type rv) r"r rrxintfloatrNZdtypernr{rlrK)rSr|rarWrXrr(r) _copy_args"  z<_insert_copy_of_node_a_after_input_node_c.._copy_argr r)r]rf)r"r r5rnrrrrroitemsr{rr!r#r$rr4r6)rrrVrarWrbrrrnew_argsZ new_kwargsrrZnew_argZ kwarg_nameZnode_a_shadows_c_nameZnew_mod_copy_nameZmod_anode_a_shadows_cr(rr)r;s\ 3                 rname_aname_bmatched_subgraph_pairsshould_log_inputsc3s(|durt}t}ifdd} i} i} |D]&\} } | \}}t|j|}t|j}|| ||f| |j<|| ||f| |j<qjjD]}|j dkrY| t |j d| qE|| v}|| v}|sf|r|rq| |\}}}}n |suJ| |\}}}}t |jot |}|stdt|dt|j|d||| |j<qEt|j|||\}}t|||\}}|tjko|tjko|tjko|tjk}|stdt|dt|j|d ||| |j<qE|tjkr$|tjkr$t|j||}|s$tdt|dt|j|d ||| |j<qEt|j|}t|||sPtdt|dt|j|d ||| |j<qEt|j|}t|j}|r|rt|d} t| tr| j}!t|!|d |j|||tjj dd|d |!j<n>t| t!rfdd| D}"t"| D]\}#}$|"|#}!t|!|d |j|||tjj |#d|d |!j<qn t#dt$| d|s|r||| |j<|j}%|rt|%d}!|rt|!trt|%d}!nt|!t!rfdd|!D}!t%|j|%|!|||jd|| }&|rcd}'t|&tr4t|&|d|'|||tjj dd|d }&|&}(n/t|&t!s op0_fp32 -> a1 -> op1_fp32 -> a2 Graph B: b0 -> op0_int8 -> b1 -> op1_int8 -> b2 matched_node_pairs: {'op0': (op0_fp32, op0_int8), 'op1': (op1_fp32, op1_int8)} Graph C (A shadows B): / dequant0 -> op0_fp32 -> logger_a_0 / dequant_1 -> op1_fp32 -> logger_a_1 / / b0 -------------> op0_int8 -> logger_b_0 --------------> op1_int8 -> logger_b_1 In a nutshell, this function does the following for each node pair: * copies the necessary attributes and modules from gm_a to gm_b, keeping names unique * adds a dtype cast op (dequant, quant, etc) * adds a copy of node_a in gm_b's graph * adds loggers to the outputs of node_a and node_b Ncr:)Ncr;r<r=r>env_cr(r)rArBz6create_a_shadows_b..load_arg..rCrDrr(r)rFrGz$create_a_shadows_b..load_argrHrz$skipping shadow loggers for node_b: z, start_node_a: z , unsupportedz, unknown dtype castz, unknown input qparamsz", unhandled logic in subgraph copyZ_ns_logger_b_inp_rIcsg|]}|jqSr(r=.0rSrr(r) sz&create_a_shadows_b..ztype z is not handled yetcsg|]}t|dqS)r)rr)rWr(r)rs Z _dtype_cast_Z_ns_logger_a_inp_rr rtZ _ns_logger_a_Z _ns_logger_b_),rr rrZ base_op_noderrr5rJr!rHr rrprintrPr%rrrjrhrgrrrr*rr"r r7rrLrMrnrOrlrKrsrorgetattrr-rQr )3rrarrWrr+rrcrXrFZ+start_node_b_to_matched_subgraph_a_and_nameZ)end_node_b_to_matched_subgraph_a_and_nameZ match_namematchryZ subgraph_bZref_node_type_aZref_node_type_bZnode_bZnode_b_is_start_nodeZnode_b_is_end_noder/Zall_op_types_support_shadowingrpZnode_output_type_aZnode_input_type_bZnode_output_type_bZnode_io_types_known_a_and_brqrzZ fqn_base_aZ fqn_base_bZ prev_node_brUZprev_node_c_listrRrSr`Zdtype_cast_noder-Z input_loggerZ new_loggersZdtype_cast_idxZdtype_cast_node_innerZdtype_cast_loggerZnode_c_second_non_param_argrrZinput_logger_modZinput_logger_innerZgm_cr()rrWr)create_a_shadows_bs8$                                      rr<)4typingrrrrrNZtorch.ao.ns.fx.mappingsrZtorch.ao.quantization.fx.utilsrZtorch.ao.quantization.observerrZtorch.fxr r Ztorch.fx.graphr r Zns_typesrrrutilsrrrrrrrrrrr$r*rr7dictr{rTZTensorrr_rnsetrsrxboolrrrrr(r(r(r)s@   0     0 o   !  ! ; K +