a hH@sZddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZddlmZddlmZmZmZmZmZmZmZddlmZmZmZddlmZerdd lm Z dd l!m"Z"m#Z#m$Z$ee%&j'Z(e(j'Z)Gd d d eZ*d Z+dddddZ,edZ-edZ.ddddddZ/ddddddZ0ej1ddd d!d"Z2erxed#d$d$d%d&d'Zej3dd(d)d*d+Z4dd,d-d.d/Z5Gd0d1d1Z6dNd2d3d1d4d5d6Z7dOd8d,d,dd9d:d;Z8dPd8d,d,d,dd<d=d>Z9d?d,d,d,dd@dAdBZ:dCd,d,d,dd@dDdEZ;dFd,d,d,ddddGdHdIZdS)Q) annotationsN)fields is_dataclass)autoEnum)Path)AnyCallableGenericLiteralNoReturn TYPE_CHECKINGTypeVar) assert_never deprecatedSelf) CodeTemplate) Namespace)IterableIteratorSequencec@s0eZdZeZeZeZeZeZeZ dS)TargetN) __name__ __module__ __qualname__rZ DEFINITIONZ DECLARATIONZ REGISTRATIONZANONYMOUS_DEFINITIONZNAMESPACED_DEFINITIONZNAMESPACED_DECLARATIONrrtd|}|dur"td||\}}}||dfS)Nz(\w+)(\.\w+)?\((.*)\)zUnsupported function schema: , )rematch RuntimeErrorgroupssplit)rmname_paramsrrrsplit_name_params7s  r*TSzCallable[[T], S | None] Iterable[T]z Iterator[S])funcxsrccs$|D]}||}|dur|VqdSNr)r.r/xrrrrmapMaybeGsr3zCallable[[T], Sequence[S]]ccs|D]}||EdHqdSr0r)r.r/r1rrr concatMapOsr4zCallable[[], str]zIterator[None])msg_fnrc cs|z dVWnltyv}zT|}t|d}|jrF|jdd|n|}|f|jdd|_WYd}~n d}~00dS)Nz r ) Exceptiontextwrapindentargs)r5emsgrrrcontextWs  r>z*Use typing_extensions.assert_never insteadr )r1rcCstdt|jdS)NzUnhandled type: )AssertionErrortyper)r1rrrrhsrr) template_fnrcCs t|Sr0)r from_file)rArrr_read_templatemsrCint)srcCs&tj|ddd}tj|ddS)Nlatin1F)usedforsecuritylittle) byteorder)hashlibsha1encodedigestrD from_bytes)rErKrrrstring_stable_hashssrOc @seZdZdddddddZeddd d Zdd dd d dZddd dddZdddddddZddddddZ ddddddddd dd!d"d#Z dddddddddd dd$ d%d&Z d ddd'd(d)Z d dd*d+Z dS), FileManagerz str | PathboolNone) install_dir template_dirdry_runrcCs&t||_t||_t|_||_dSr0)rrSrTsetfilesrU)selfrSrTrUrrr__init__|s  zFileManager.__init__zfrozenset[str]rcCstdd|jDS)NcSsh|] }|qSras_posix.0filerrr z(FileManager.filenames..) frozensetrWrXrrr filenamesszFileManager.filenamesr)filenamecontentsrcCsZt|}d}z|jdd}Wnty.Yn0||krV|jjddd|j|dddS)Nzutf-8)encodingT)parentsexist_ok)r read_textOSErrorparentmkdir write_text)rXrerfr_Z old_contentsrrr_write_if_changeds zFileManager._write_if_changedz"Callable[[], str | dict[str, Any]])rA env_callablerc Cst|rJd||j|}|}t|trd|vrtd}zttjdjpV| }Wnt t fy~| }Yn0z| t }Wnty|j}Yn0i|ddd|d|i}t|}||} tjdd | tjd St|tr|St|dS) Nztemplate_fn must be relative: Zgenerated_commentzgen.py__main__@z generated by z from z (""")\n+ # match triple quotes ( (\s*@.+\n)* # match decorators if any \s*(class|def) # match class/function definition ) z\g<1>\n\n\g<2>)flags)r is_absoluterT isinstancedict TORCHGEN_ROOTsysmodules__file__absoluteKeyErrorAttributeError relative_to REPO_ROOTr\ ValueErrorr'rC substituter!subVERBOSErr) rXrArpZ template_pathenvZgenerator_default generatorZgenerator_pathtemplatesubstitute_outrrrsubstitute_with_templatesF       z$FileManager.substitute_with_template)rerArprcCsrt|}|rJd||j|}||jvs@Jd||j||jsn|j||d}|j||ddS)Nfilename must be relative: zduplicate file write )rArp)rerf)rrtrSrWaddrUrro)rXrerArpr_rrrrwrite_with_templates  zFileManager.write_with_template)rerprcCs||||dSr0)r)rXrerprrrwriteszFileManager.writeN)base_envr-zCallable[[T], str]z#Callable[[T], dict[str, list[str]]]rDzdict[str, Any] | Nonezset[str])reitemskey_fnrp num_shardsr sharded_keysrc Cs|j||||||||ddS)N)rrprrr)write_sharded_with_template)rXrerrrprrrrrr write_shardeds zFileManager.write_sharded) rerArrrprrrrcsft|} | rJd|ddi} ddt|D} | g| } |dur^| D]|qND]F} | D]<| vrt| tsJd| | <qjg| <qjqbdddd fd d }|jrd }|D]8}||} t| |}||}|| |||| |q| D]4d}| | | j ||fd dq |j |j| | j ddS)Nrshard_idZ EverythingcSsg|]}dd|iqS)rr(r)r^irrr sz;FileManager.write_sharded_with_template..z'sharded keys in base_env must be a listzdict[str, list[str]]rR)intofrom_rcs<|D].\}}|vs&Jd||||7<qdS)Nzundeclared sharded key )r)rrkv)rrr merge_envsz:FileManager.write_sharded_with_template..merge_envrcsSr0rr)shardrr5raz9FileManager.write_sharded_with_template..)rrtrangeupdaterulistcopyrUrOr with_stemstemrWdiscardrS)rXrerArrrprrrr_Z everythingZshardsZ all_shardskeyritemZsidrrr)rrrrsF       z'FileManager.write_sharded_with_template) variable_namerercCs8dd|gddt|jDdR}|||dS)zSWrite a file containing the list of all outputs which are generated by this script.r6zset(css|]}d|dVqdS)z ""Nr[r]rrr Braz,FileManager.write_outputs..)N)joinsortedrWro)rXrrecontentrrr write_outputs;s zFileManager.write_outputscCstj|jtjtS)z This needs to be deterministic. The template dir is an absolute path that varies across builds. So, just use the path relative to this file, which will point to the codegen source but will be stable. )ospathrelpathrTdirnamerzrcrrrtemplate_dir_for_commentsHsz%FileManager.template_dir_for_comments)rrrrYpropertyrdrorrrrrrrrrrrrP{s  6 "> rPrzstr | Path | None)optionsrSrcCs.tj|jd}|r|n|j}t|||jdS)NZ templates)rSrTrU)rrr source_pathrSrPrU)rrSrTrrrmake_file_managerRsrPr)objr:widthrcCs0tjdkr"ddlm}||||St|||dS)N) r)pformat)r:r)rx version_infopprintr_pformat)rr:rrrrrdataclass_repr`s   r)rr:r curr_indentrc stsJdtjj}|t|d7}fddtD}g}|D]\}}|t|d} t|rt|||| } nBt|trt |||| } n(t|t t t frt |||| } nt|} ||d| qP|d} d| |} |d| d S) Nz%obj should be a dataclass, received: r7cs$g|]}|jr|jt|jfqSr)reprr'getattr)r^frrrrzraz_pformat..= z, (r)rr@ __class__rlenrrrurv _format_dictrrVtuple _format_listrappendr) rr:rr class_nameZ fields_list fields_strr'attrZ _curr_indentZstr_repr indent_strbodyrrrrns$  rzdict[Any, Any])rr:rrrc Csv||d7}g}|D]J\}}t|}t|rFt||||t|nt|}||d|qt||||ddS)Nrz: {})rrrrrr_format) rr:rrZ dict_reprrrZk_reprZv_strrrrrs rz&list[Any] | set[Any] | tuple[Any, ...]csJd7fdd|D}t|tr0dnd\}}t|||S)Nr7cs*g|]"}t|rt|nt|qSr)rrr)r^lrr:rrrrsz _format_list..)[])rr)rurr)rr:rrZ list_reprstartendrrrrs  rz list[str])rr:rrrrrc CsTd\}}tt||kr$d}d|}d|}d|||} ||| |S)N)rr6rr )rrr) rr:rrrr delimiterZcurr_indent_strrrrrrrsrc@seZdZdZdddddddd Zeddddd d d Zedd ddZedd ddZ edd ddZ ddddddZ dS)NamespaceHelperaA helper for constructing the namespace open and close strings for a nested set of namespaces. e.g. for namespace_str torch::lazy, prologue: namespace torch { namespace lazy { epilogue: } // namespace lazy } // namespace torch rrrDrR) namespace_str entity_name max_levelrcCsr|d}t||ks,Jd|d|d||_ddd|D|_dddt|D|_||_||_dS) N::z"Codegen doesn't support more than z# level(s) of custom namespace. Got .r6cSsg|]}d|dqS)z namespace z {rr^nrrrrraz,NamespaceHelper.__init__..cSsg|]}d|qS)z} // namespace rrrrrrra) r%rcpp_namespace_r prologue_reversed epilogue_Z namespaces_ entity_name_)rXrrrZcpp_namespacesrrrrYs zNamespaceHelper.__init__)namespaced_entityrrcCs2|d}|d}d|dd}t|||dS)zq Generate helper from nested namespaces as long as class/function name. E.g.: "torch::lazy::add" rN)rrr)r%rr)rrnamesrrrrrfrom_namespaced_entitys  z&NamespaceHelper.from_namespaced_entityrZcCs|jSr0)rrcrrrprologueszNamespaceHelper.prologuecCs|jSr0)rrcrrrepilogueszNamespaceHelper.epiloguecCs|jSr0)rrcrrrrszNamespaceHelper.entity_name)defaultrcCs|jr |jS|S)z Return the namespace string from joining all the namespaces by "::" (hence no leading "::"). Return default if namespace string is empty. )r)rXrrrrget_cpp_namespacesz!NamespaceHelper.get_cpp_namespaceN)rr)r)r) rrr__doc__rY staticmethodrrrrrrrrrrrsrc@seZdZUded<d&dddddZd d d d d ZddddZdddddZd dd ddZddddZ e dddddZ dddddZ dd dd!d"Z d#d dd$d%ZdS)' OrderedSetzdict[T, Literal[None]]storageNzIterable[T] | NonerR)iterablercCs |duri|_n t||_dSr0)rrvfromkeys)rXrrrrrYszOrderedSet.__init__r+rQ)rrcCs ||jvSr0rrXrrrr __contains__szOrderedSet.__contains__z Iterator[T]rZcCst|jSr0)iterrkeysrcrrr__iter__szOrderedSet.__iter__z OrderedSet[T])rrcCs|j|jdSr0)rr)rXrrrrrszOrderedSet.updatecCsd|j|<dSr0rrrrrrszOrderedSet.addcCst}|j|_|Sr0)rrr)rXretrrrr!s zOrderedSet.copy)r;rcGs,|d}|ddD]}||q|S)Nrr7)rr)r;rrErrrunion&s  zOrderedSet.union)otherrcCs t||Sr0)rrrXrrrr__or__-szOrderedSet.__or__rcCs|||Sr0)rrrrr__ior__0s zOrderedSet.__ior__objectcCs,t|tr|j|jkSt|j|kSdSr0)rurrrVrrrrr__eq__4s  zOrderedSet.__eq__)N)rrr__annotations__rYrrrrrrrrrr rrrrr s r)N)rr)r)? __future__r contextlib functoolsrJrr!rxr9Z dataclassesrrenumrrpathlibrtypingrr r r r r rZtyping_extensionsrrrZtorchgen.code_templaterargparsercollections.abcrrrrzr{rlrwrrZ IDENT_REGEXr*r+r,r3r4contextmanagerr>cacherCrOrPrrrrrrrrrrrrs^  $    Z#E