o
    Zh                     @   s  d Z ddlZddlZddlmZmZ ddlZddlm	Z	 ddl
mZ eeZddd	Zi d
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+i d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMi dNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtZdudv ZG dwdx dxe	ZdxgZdS )yz)Tokenization classes for Salesforce CTRL.    N)OptionalTuple   )PreTrainedTokenizer)loggingz
vocab.jsonz
merges.txt)
vocab_filemerges_fileZ	Pregnancyi ZChristianityi  ZExplaini ZFitnessi  ZSavingi  ZAski#j  ZAssiv ZJokei~ Z	Questionsi6  ZThoughtsi  ZRetailiv  ZFeminismi ZWritingi.  ZAtheismi ZNetflixi  Z	Computingiך  ZOpinioniͨ  ZAlonei  Funnyi%  ZGamingi  ZHumani  ZIndiai3  ZJokeriR- ZDietin  ZLegaliS.  ZNormaniK  ZTipi ZWeightiw  ZMoviesi  ZRunningi[  ZSciencei*  ZHorrori  Z
Confessioni  ZFinancei/  ZPoliticsi?  ZScaryi ZSupportin1  ZTechnologiesi  ZTeenageip Eventi  ZLearnedi ZNotioni Z	Wikipediaiϒ  ZBooksi	  ZExtracti) ZConfessionsi- Z
Conspiracyi( ZLinksi  Z	NarcissusiK ZRelationshipi  ZRelationshipsi iǢ  i  ih  i )ZReviewsZNewsZTranslationZmultilingualc                 C   s>   t  }| d }| dd D ]}|||f |}qt |}|S )z
    Return set of symbol pairs in a word.

    Word is represented as tuple of symbols (symbols being variable-length strings).
    r      N)setadd)wordpairsZ	prev_charchar r   Y/var/www/auris/lib/python3.10/site-packages/transformers/models/ctrl/tokenization_ctrl.py	get_pairs^   s   r   c                       s   e Zd ZdZeZeZd fdd	Ze	dd Z
dd Zd	d
 Zdd Zdd Zdd Zdd Zddedee dee fddZ  ZS )CTRLTokenizera`  
    Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    <unk>c                    s   t |dd}t|| _W d    n1 sw   Y  dd | j D | _t |dd}| ddd }W d    n1 sCw   Y  dd	 |D }tt	|t
t|| _i | _t jdd
|i| d S )Nutf-8encodingc                 S   s   i | ]\}}||qS r   r   ).0kvr   r   r   
<dictcomp>   s    z*CTRLTokenizer.__init__.<locals>.<dictcomp>
r   c                 S   s   g | ]}t | qS r   )tuplesplit)r   merger   r   r   
<listcomp>   s    z*CTRLTokenizer.__init__.<locals>.<listcomp>	unk_tokenr   )openjsonloadencoderitemsdecoderreadr    dictziprangelen	bpe_rankscachesuper__init__)selfr   r   r#   kwargsZvocab_handleZmerges_handleZmerges	__class__r   r   r2      s   zCTRLTokenizer.__init__c                 C   s
   t | jS N)r.   r'   r3   r   r   r   
vocab_size   s   
zCTRLTokenizer.vocab_sizec                 C   s   t | jfi | jS r7   )r+   r'   Zadded_tokens_encoderr8   r   r   r   	get_vocab   s   zCTRLTokenizer.get_vocabc           
         s  | j v r
 j | S t|}tt|d d |d d g }t|}|s'|S 	 t| fddd}| jvr8ny|\}}g }d}|t|k rz|||}	W n ty`   |	||d   Y n?w |	|||	  |	}|| |kr|t|d k r||d  |kr|
||  |d	7 }n|
||  |d7 }|t|k sFt|}|}t|dkrnt|}q(d
|}|d d }| j |< |S )Nr   z</w>Tc                    s    j | tdS )Ninf)r/   getfloat)pairr8   r   r   <lambda>   s    z#CTRLTokenizer.bpe.<locals>.<lambda>keyr   r      @@ )r0   r   listr   minr/   r.   index
ValueErrorextendappendjoin)
r3   tokenr   r   ZbigramfirstsecondZnew_wordijr   r8   r   bpe   sN   

"
,


zCTRLTokenizer.bpec                 C   s8   g }t d|}|D ]}|t| |d q
|S )zTokenize a string.z\S+\n? )refindallrI   rE   rQ   r    )r3   textZsplit_tokenswordsrL   r   r   r   	_tokenize   s
   zCTRLTokenizer._tokenizec                 C   s   | j || j | jS )z0Converts a token (str) in an id using the vocab.)r'   r<   r#   )r3   rL   r   r   r   _convert_token_to_id   s   z"CTRLTokenizer._convert_token_to_idc                 C   s   | j || jS )z=Converts an index (integer) in a token (str) using the vocab.)r)   r<   r#   )r3   rG   r   r   r   _convert_id_to_token   s   z"CTRLTokenizer._convert_id_to_tokenc                 C   s   d |dd }|S )z:Converts a sequence of tokens (string) in a single string.rR   rC    )rK   replacestrip)r3   tokensZ
out_stringr   r   r   convert_tokens_to_string   s   z&CTRLTokenizer.convert_tokens_to_stringNsave_directoryfilename_prefixreturnc           
   	   C   sV  t j|std| d d S t j||r|d ndtd  }t j||r,|d ndtd  }t|ddd	}|t	j
| jd
dddd  W d    n1 sTw   Y  d}t|ddd	=}|d t| j dd dD ]!\}}	||	krtd| d |	}|d|d  |d7 }qsW d    ||fS 1 sw   Y  ||fS )NzVocabulary path (z) should be a directory-rZ   r   r   wr   r   rB   TF)indent	sort_keysensure_asciir   r   z#version: 0.2
c                 S   s   | d S )Nr   r   )kvr   r   r   r?      s    z/CTRLTokenizer.save_vocabulary.<locals>.<lambda>r@   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rR   r   )ospathisdirloggererrorrK   VOCAB_FILES_NAMESr$   writer%   dumpsr'   sortedr/   r(   warning)
r3   r_   r`   r   Z
merge_filefrG   writerZ
bpe_tokensZtoken_indexr   r   r   save_vocabulary   s8    



zCTRLTokenizer.save_vocabulary)r   r7   )__name__
__module____qualname____doc__rm   Zvocab_files_namesCONTROL_CODEScontrol_codesr2   propertyr9   r:   rQ   rW   rX   rY   r^   strr   r   rt   __classcell__r   r   r5   r   r   n   s    
,
(r   )rx   r%   rh   typingr   r   regexrS   Ztokenization_utilsr   utilsr   Z
get_loggerru   rk   rm   ry   r   r   __all__r   r   r   r   <module>   s   
	
 !"#$%&'()*+,-./01234; 
