o Zh &@sdZddlZddlZddlmZmZmZddlmZm Z er#ddl Z ddl m Z ddl mZeeZdd iZd d ZGd d d ZGddde ZdgZdS)z Tokenization classes for CPMAnt.N)ListOptionalTuple)is_jieba_availablerequires_backends)PreTrainedTokenizer)logging vocab_filez vocab.txtcCsft}t|ddd }|}Wdn1swYt|D] \}}|d}|||<q#|S)z*Loads a vocabulary file into a dictionary.rutf-8encodingN ) collections OrderedDictopen readlines enumeraterstrip)r vocabreadertokensindextokenr]/var/www/auris/lib/python3.10/site-packages/transformers/models/cpmant/tokenization_cpmant.py load_vocab$s   rc@seZdZdddZddZdS) WordpieceTokenizercCs||_||_||_dSN)r unk_tokenmax_input_chars_per_word)selfrr"r#rrr__init__0s zWordpieceTokenizer.__init__cCst|}t||jkr|jgSd}g}|t|krXt|}d}||kr"`): The beginning of document token. eod_token (`str`, *optional*, defaults to `""`): The end of document token. bos_token (`str`, *optional*, defaults to `""`): The beginning of sequence token. eos_token (`str`, *optional*, defaults to `""`): The end of sequence token. pad_token (`str`, *optional*, defaults to `""`): The token used for padding. unk_token (`str`, *optional*, defaults to `""`): The unknown token. line_token (`str`, *optional*, defaults to `""`): The line token. space_token (`str`, *optional*, defaults to `""`): The space token. Z input_idsZattention_maskFrleftc st|dg||_||_t||_|j| |jd<|j||jd<|j| =|j|=tt|jddd|_dd|jD|_ t |j|d |_ t j d |||||||| | d | dS) Njieba rcS|dSNr'rxrrrz*CpmAntTokenizer.__init__..keycSsi|]\}}||qSrr).0kvrrr sz,CpmAntTokenizer.__init__..)rr") bod_token eod_token bos_token eos_token pad_tokenr" line_token space_token padding_sider)rrKrLrencoderrrsorteditemsdecoderrwordpiece_tokenizersuperr%) r$r rKrLrMrNrOr"rPrQrRkwargs __class__rrr%ls0   zCpmAntTokenizer.__init__cC |j|jSr!)rSrKr$rrr bod_token_id zCpmAntTokenizer.bod_token_idcCr\r!)rSrLr]rrr eod_token_idr_zCpmAntTokenizer.eod_token_idcCs |jdS)NrrSr]rrr newline_id zCpmAntTokenizer.newline_idreturncCs t|jSr!)r)rSr]rrr vocab_sizerczCpmAntTokenizer.vocab_sizecCst|jfi|jSr!)dictrSZadded_tokens_encoderr]rrr get_vocabszCpmAntTokenizer.get_vocabcCs.g}tj|ddD] }||j|q |S)zTokenize a string.F)Zcut_all)r=cutextendrWr0)r$textZ output_tokensrBrrr _tokenizeszCpmAntTokenizer._tokenizec s4dd|D}fdd|D}tj|fi|S)zDecode ids into a string.cSsg|]}|dkr|qS)rr)rGirrr sz+CpmAntTokenizer._decode..cs.g|]}|jkr|jkr|jkr|qSr)Z pad_token_idZ eos_token_id bos_token_id)rGrBr]rrrms()rX_decode)r$Z token_idsrYrZr]rros  zCpmAntTokenizer._decodecCs ||jvSr!rar$rrrrcheck zCpmAntTokenizer.checkrcCs d|S)Nr&)r*)r$rrrrconvert_tokens_to_stringrrz(CpmAntTokenizer.convert_tokens_to_stringcCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)rSgetr"rprrr_convert_token_to_idsz$CpmAntTokenizer._convert_token_to_idcCs|j||jS)z=Converts an index (integer) in a token (str) using the vocab.)rVrtr")r$rrrr_convert_id_to_tokensz$CpmAntTokenizer._convert_id_to_tokenNsave_directoryfilename_prefixcCs*tj|rtj||r|dndtd}n |r|dnd|}d}d|jvr5|jd|jd<|jd=d|jvrF|jd|jd<|jd=tt|j d d d |_t |d d d.}|j D]\}}||krut d|d|}| |d|d7}qbWd|fS1swY|fS)N-r&r rr>r;rr:cSr?r@rrArrrrCrDz1CpmAntTokenizer.save_vocabulary..rEwr r zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r')ospathisdirr*VOCAB_FILES_NAMESrSrrrTrUrloggerwarningwrite)r$rwrxr rwriterrZ token_indexrrrsave_vocabularys6        zCpmAntTokenizer.save_vocabulary token_ids_0 token_ids_1cCs,|dur |jg|S|jg||jg|S)a1 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A CPMAnt sequence has the following format: - single sequence: `[BOS] Sequence`. Args: token_ids_0 (`List[int]`): The first tokenized sequence that special tokens will be added. token_ids_1 (`List[int]`): The optional second tokenized sequence that special tokens will be added. Returns: `List[int]`: The model input with special tokens. N)rn)r$rrrrr build_inputs_with_special_tokenss z0CpmAntTokenizer.build_inputs_with_special_tokensalready_has_special_tokenscsZ|r tj||ddS|dur#dgdgt|dgdgt|Sdgdgt|S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rrrNr'r)rXget_special_tokens_maskr))r$rrrrZrrrs(z'CpmAntTokenizer.get_special_tokens_mask) r5r6r7r8r9rr:r;r<r!)NF)r1r2r3__doc__r~Zvocab_files_namesZmodel_input_namesZadd_prefix_spacer%propertyr^r`rbintrergrkrorqrstrrsrurvrrrrboolr __classcell__rrrZrr4Osb*        r4)rrr{typingrrrZtransformers.utilsrrr=Ztokenization_utilsrutilsr Z get_loggerr1rr~rrr4__all__rrrrs       B