
    /h2                        S SK r S SKJr  S SKJr  S SKJr  \" SS/S9r\ R                  " \S9\ R                  " 5       S	 5       5       r
\
R                  S
5      \ R                  " SSSSS9\ R                  " SSSSSS9\ R                  " SSSSS9\ R                  " SSSSS9\ R                  " SSSSS9S  5       5       5       5       5       5       rg)!    N)tqdm)word_tokenize)parallelize_preprocessz-hz--help)help_option_names)context_settingsc                      g )N r	       @/var/www/auris/envauris/lib/python3.13/site-packages/nltk/cli.pyclir      s     	r
   tokenizez
--languagez-lenz1The language for the Punkt sentence tokenization.)defaulthelpz--preserve-lineTzIAn option to keep the preserve the sentence and not sentence tokenize it.)r   is_flagr   z--processesz-j   zNo. of processes.z
--encodingz-eutf8zSpecify encoding of file.z--delimiterz-d z%Specify delimiter to join the tokens.c           
         [         R                  " SUS9 n[         R                  " SUS9 nUS:X  aD  [        UR                  5       5       H&  n[	        UR                  [        U5      5      SUS9  M(     O?[        [        UR                  5       USS9 H  n[	        UR                  U5      SUS9  M     S	S	S	5        S	S	S	5        g	! , (       d  f       N= f! , (       d  f       g	= f)
z;This command tokenizes text stream using nltk.word_tokenizestdin)encodingstdoutr   
)endfileT)progress_barN)clickget_text_streamr   	readlinesprintjoinr   r   )	languagepreserve_line	processesr   	delimiterfinfoutlineoutlines	            r   tokenize_filer*      s    * 
		w	:c""8h?4 A~ 1D)..t)<=4dS 2  6!3==?ID G )..1t$G  @ 
;	:?? 
;	:s#   CB
C6C
C	C
C&)r   r   nltkr   	nltk.utilr   dictCONTEXT_SETTINGSgroupversion_optionr   commandoptionr*   r	   r
   r   <module>r3      s       ,4*:;  ./	  0	 Z	<	 	T mT13FGlD&7RS4+RH T H &Hr
   