o wZh*@sfUddlZddlZddlmZddlmZmZdeede defddZ d ed ed ed ee eee ffd dZ deede ede efddZdedefddZdedefddZdede eeefeeefe eeeeefffddZde ede eeefeeefe eeeeefffddZd>dededeefd d!Zdedefd"d#Z $d?deeeefde ed%e ed&ed'eeefd(edeefd)d*Zd+Zeed,<d-d.hZe eed/<e Ze eed0<d1d1d2Zeeefed3<d4Zeed5<d-d.hZe eed6<e Ze eed7<d8d9iZeeefed:<d@d;d<Z e!d=kr1e dSdS)AN) defaultdict)AnyUnionlines indentationreturncCsFd}dd|}t|D]\}}|dkr||7}||d|7}q |S)N  r) enumeratereplace)rroutputZnew_line_with_indentilinerQ/var/www/auris/lib/python3.10/site-packages/torch/utils/data/datapipes/gen_pyi.pymaterialize_liness rdir template_name output_name replacementsc Cstj||}tj||}t| }|}Wdn1s!wY|D](\}} } t|d}||t| | }||Wdn1sKwYq(dS)Nw)ospathjoinopenreadr rwrite) rrrrZ template_pathZ output_pathfcontent placeholderrrrrrgen_from_templates     r! dir_pathsfiles_to_excludecsVt}|D]#t}dd|D}fdd|D}|fdd|Dq|S)z When given a path to a directory, returns the paths to the relevant files within it. This function does NOT recursive traverse to subdirectories. cSs h|] }d|ddkr|qS)z.pyNr.0fnamerrr .s z"find_file_paths..csh|]}|vr|qSrrr%r#rrr(/scsh|] }tj|qSrrrrr%)dir_pathrrr(2)setrlistdirupdate)r"r#paths all_filesZ python_filesZ filter_filesr)r+r#rfind_file_paths%s  r2rcCs\d|vr d\}}nd|vrd\}}ntd|||t|||}}|||S)zXExtract method name from decorator in the form of "@functional_datapipe({method_name})".(")r3z")(')r4z')z4Unable to find appropriate method name within line: ) RuntimeErrorfindlenrZ start_tokenZ end_tokenstartendrrrextract_method_name6s   r;cCs2d}d}||t|||}}|||S)zVExtract class name from class definition in the form of "class {CLASS_NAME}({Type}):".class ()r6r7r8rrrextract_class_nameDs r> file_pathcCsiit}}}tt}t|}d}d\}}} d} |D]} | dddkr+| } | s1d| vr9||| qd| vrFt| }g||<q|rQd| vrQt| }q|rwd | vs[d | vrwd | vrd|||d7}| d t d } | | d } |dkr|| d 7}|| d 8}|dkr| d } | | d | 7} t | ||<|||<d\}}} q|dkrt d| | dd7} qWd n1swY||||fS)zfGiven a path to file, parses the file and returns a dictionary of method names to function signatures.r)rrrFz"""z@functional_datapiper<z def __init__(z def __new__(r=N)z7open parenthesis count < 0. This shouldn't be possible.r r )r-rlistrcountappendr;r>addr6r7rfindprocess_signaturer5strip)r?method_to_signaturemethod_to_class_nameZspecial_output_typedoc_string_dictrZopen_paren_count method_name class_name signatureskiprr9r:rrrparse_datapipe_fileLs^         'rQ file_pathsc Csdiit}}}i}|D]}t|\}}}} |||||||| q ||||fS)N)r-rQr/) rRmethods_and_signaturesmethods_and_class_namesZ!methods_with_special_output_typesmethods_and_doc_stringsrrJrKZ$methods_needing_special_output_typesrLrrrparse_datapipe_filess,     rV, delimitercCspd}d}g}|D](}|dkr|d7}n|dkr|d8}n||kr,|dkr,||d}q||7}q|||S)zRGiven a line of text, split it on comma unless the comma is within a bracket '[]'.rr[rA])rE)rrXZ bracket_countZ curr_tokenrescharrrrsplit_outside_brackets     r]cCst|}t|D]B\}}|d||<|dkrd||<q|dkr5d||dkr5||ddkr5d||<qd|vrJ|d d \}}|dd ||<qd d |D}d|}|S)z Clean up a given raw function signature. This includes removing the self-referential datapipe argument, default arguments of input functions, newlines, and spaces. r clsselfrrA*rz Callable ==r@z= ...cSsg|]}|dkr|qSrr)r&trrr sz%process_signature..z, )r]r rIrsplitr)rtokensrtokenheadZ _default_argrrrrHs (  rHrdeprecated_filesdefault_output_typemethod_to_special_output_typerootc s dkrtttjt|tr|gn|}fdd|D}t|||d}t |\}}} } |D] } | | vr?| | q4g} | D]4\} }|| }| | vrW|| }n|}d | | }|dkrfd}| d|d| d|d |d | qF| jd d d | S)a #.pyi generation for functional DataPipes Process. # 1. Find files that we want to process (exclude the ones who don't) # 2. Parse method name and signature # 3. Remove first argument after self (unless it is "*datapipes"), default args, and spaces rcsg|] }tj|qSrr*)r&rrlrrrdr,z*get_method_definitions..r)z ... z# Functional form of 'z' def r=z) -> z: cSs|ddS)Nr rA)split)srrrsz(get_method_definitions..)key)strpathlibPath__file__parentresolve isinstancer2unionrVrFitemsrrEsort)r?r#rirjrkrlrRrSrTZmethods_w_special_output_typesrUfn_nameZmethod_definitionsrM argumentsrN output_typeZ doc_stringrrmrget_method_definitionssR   riteriterDP_file_pathz __init__.pyzutils.pyiterDP_files_to_excludeiterDP_deprecated_fileszList[IterDataPipe])Zdemuxfork$iterDP_method_to_special_output_typemapmapDP_file_pathmapDP_files_to_excludemapDP_deprecated_filesshuffle IterDataPipe#mapDP_method_to_special_output_typecCs\ttttdt}ttttdt}t t j }d|dfd|dfg}tt|dd|dd S) a # Inject file into template datapipe.pyi.in. TODO: The current implementation of this script only generates interfaces for built-in methods. To generate interface for user-defined DataPipes, consider changing `IterDataPipe.register_datapipe_as_function`. rZ MapDataPipez${IterDataPipeMethods}z${MapDataPipeMethods}zdatapipe.pyi.inz datapipe.pyi)rrrrN)rrrrrrrrrrsrtrurvrwr!rr)Ziter_method_definitionsZmap_method_definitionsrrrrrmain s0 r__main__)rWrb)rN)"rrs collectionsrtypingrrrCrrintrtupler!r-r2r;r>dictrQrVr]rHrr__annotations__rrrrrrrr__name__rrrrsv    ". 4.   8   $