
    fThx                     &   % S r SSKrSSKrSSKrSSKrSSKJr  SSKJrJ	r	J
r
JrJr  SSKJr  SSKJrJr  SSKJr  SS	KJr  SS
KJr  SSKJrJrJrJrJrJr  SSKJ r   SSK!J"r"  SSK#J$r$J%r%J&r&J'r'J(r(  \" 5       (       a  SSK)J*r*  OSr*\RV                  " \,5      r-\(       a  \" 5       r.\\/\\
\/   \
\/   4   4   \0S'   GO\" / S\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSPS \" 5       (       a  S!OS\" 5       (       a  S"OS44PS#PS$S\" 5       (       a  SOS44PS%\" 5       (       a  S&OSS44PS'PS(PS)\" 5       (       a  S*OS\" 5       (       a  S+OS44PS,S-\" 5       (       a  S.OS44PS/PS0S\" 5       (       a  S1OS44PS2PS3PS4S\" 5       (       a  SOS44PS5S6\" 5       (       a  S7OS44PS8S\" 5       (       a  S9OS44PS:S;\" 5       (       a  S<OS44PS=S\" 5       (       a  SOS44PS>PS?\" 5       (       a  S@OS\" 5       (       a  SAOS44PSBPSC\" 5       (       a  SOS\" 5       (       a  SOS44PSDS\" 5       (       a  SOS44PSES;\" 5       (       a  S<OS44PSFSG\" 5       (       a  SHOS44PSISG\" 5       (       a  SHOS44PSJPSK\" 5       (       a  SLOS\" 5       (       a  SMOS44PSNSO\" 5       (       a  SPOS44PSQS\" 5       (       a  SOS44PSRS\" 5       (       a  SOS44PSSS\" 5       (       a  SOS44PSTSU\" 5       (       a  SVOS44PSW\" 5       (       a  SXOS\" 5       (       a  SYOS44PSZPS[PS\PS]S;\" 5       (       a  S<OS44PS^S6\" 5       (       a  S7OS44PS_S`\" 5       (       a  SaOS44PSb\" 5       (       a  ScOS\" 5       (       a  SdOS44PSe\" 5       (       a  SOS\" 5       (       a  SOS44PSf\" 5       (       a  SOS\" 5       (       a  SOS44PSgSh\" 5       (       a  SiOS44PSjSk\" 5       (       a  SlOS44PSmSn\" 5       (       a  SoOS44PSpS6\" 5       (       a  S7OS44PSqS\" 5       (       a  SOS44PSr\" 5       (       a  SsOSS44PStPSuS\" 5       (       a  S1OS44PSvS\" 5       (       a  SwOS44PSx\" 5       (       a  SyOSS44PSzPS{S|\" 5       (       a  S}OS44PS~PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  S1OS44PSS\" 5       (       a  S1OS44PS\" 5       (       a  SOSS44PSS6\" 5       (       a  S7OS44PSS6\" 5       (       a  S7OS44PSS6\" 5       (       a  S7OS44PSS\" 5       (       a  SwOS44PSPSS6\" 5       (       a  S7OS44PSPSS\" 5       (       a  SOS44PSSG\" 5       (       a  SHOS44PSS\" 5       (       a  S1OS44PSS\" 5       (       a  SOS44PSPSS;\" 5       (       a  S<OS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS6\" 5       (       a  S7OS44PSS6\" 5       (       a  S7OS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSPS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSPSS\" 5       (       a  SOS44PS\" 5       (       a  SOSS44PSS\" 5       (       a  SwOS44PSS\" 5       (       a  SwOS44PS\" 5       (       a  SOSS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS;\" 5       (       a  S<OS44PSS\" 5       (       a  SOS44PSPS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOSS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  S1OS44PSS\" 5       (       a  S1OS44PSS\" 5       (       a  S1OS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SwOS44PSS;\" 5       (       a  S<OS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSS\" 5       (       a  SOS44PSPSS\" 5       (       a  S1OS44PSS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PS\" 5       (       a  SOS\" 5       (       a  SOS44PSS\" 5       (       a  SwOS44PSS\" 5       (       a  SwOS44PSS\" 5       (       a  SwOS44PSSG\" 5       (       a  SHOS44PSSG\" 5       (       a  SHOS44PGS GS\" 5       (       a  GSOS44PGSS6\" 5       (       a  S7OS44PGSSG\" 5       (       a  SHOS44PGSSG\" 5       (       a  SHOS44PGSS\" 5       (       a  SOS44PGS\" 5       (       a  S-OS\" 5       (       a  S.OS44PGS\" 5       (       a  S-OS\" 5       (       a  S.OS44PGS	PGS
\" 5       (       a  SOS\" 5       (       a  SOS44PGSSO\" 5       (       a  SPOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSPGSS\" 5       (       a  SOS44PGSS\" 5       (       a  S1OS44PGS\" 5       (       a  GSOSS44PGSPGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSS\" 5       (       a  SOS44PGSPGSGS\" 5       (       a  GS OS44PGS!\" 5       (       a  SOS\" 5       (       a  SOS44PGS"\" 5       (       a  GS#OS\" 5       (       a  GS$OS44PGS%\" 5       (       a  GS&OS\" 5       (       a  GS'OS44PGS(GS)\" 5       (       a  GS*OS44PGS+S;\" 5       (       a  S<OS44PGS,S;\" 5       (       a  S<OS44PGS-PGS.GS/\" 5       (       a  GS0OS44PGS1S\" 5       (       a  SwOS44PGS2\" 5       (       a  GS3OS\" 5       (       a  GS4OS44PGS5\" 5       (       a  GS3OS\" 5       (       a  GS4OS44PGS6\" 5       (       a  SOS\" 5       (       a  SOS44PGS7\" 5       (       a  GS8OSS44PGS9\" 5       (       a  SOS\" 5       (       a  SOS44PGS:\" 5       (       a  GS;OSS44PGS<PGS=\" 5       (       a  GS>OSS44PGS?PGS@GSA\" 5       (       a  GSBOS44PGSCS\" 5       (       a  SwOS44PGSDS6\" 5       (       a  S7OS44PGSE\" 5       (       a  SOS\" 5       (       a  SOS44PGSF\" 5       (       a  SOS\" 5       (       a  SOS44PGSGPGSHPGSIPGSJS\" 5       (       a  SOS44PGSK\" 5       (       a  GSLOS\" 5       (       a  GSMOS44PGSN\" 5       (       a  SOS\" 5       (       a  SOS44PGSOS\" 5       (       a  SOS44PGSPS\" 5       (       a  SOS44PGSQS\" 5       (       a  SOS44PGSRS\" 5       (       a  SOS44PGSSPGSTPGSUPGSVPGSWPGSXGSY\" 5       (       a  GSZOS44PGS[SG\" 5       (       a  SHOS44PGS\\" 5       (       a  GS]OS\" 5       (       a  GS^OS44PGS_PGS`\" 5       (       a  GSaOSS44PGSb\" 5       (       a  SOS\" 5       (       a  SOS44PGSc\" 5       (       a  SOS\" 5       (       a  SOS44PGSd\" 5       (       a  GSeOS\" 5       (       a  GSfOS44PGSg\" 5       (       a  SOS\" 5       (       a  SOS44PGSh\" 5       (       a  SOS\" 5       (       a  SOS44PGSi\" 5       (       a  SOS\" 5       (       a  SOS44PGSj\" 5       (       a  SOS\" 5       (       a  SOS44P5      r.\"" \$\.5      r1\$Rd                  " 5        V Vs0 s H  u  pX_M	     snn r3GSk\/4GSl jr4        GSzGSm\\/\Rj                  4   GSn\
\\/\Rj                  4      GSo\6GSp\
\6   GSq\
\	\/\/4      GSr\
\\6\/4      GSs\
\/   GSt\6GSu\/4GSv jjr7 " GSw GSx5      r8GSyGSx/r9gs  snn f ({  zAuto Tokenizer class.    N)OrderedDict)TYPE_CHECKINGDictOptionalTupleUnion   )PretrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)load_gguf_checkpoint)PreTrainedTokenizer)TOKENIZER_CONFIG_FILE)cached_fileextract_commit_hashis_g2p_en_availableis_sentencepiece_availableis_tokenizers_availablelogging   )EncoderDecoderConfig   )_LazyAutoMapping)CONFIG_MAPPING_NAMES
AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)PreTrainedTokenizerFastTOKENIZER_MAPPING_NAMESalbertAlbertTokenizerAlbertTokenizerFastalignBertTokenizerBertTokenizerFastariaLlamaTokenizerLlamaTokenizerFast
aya_visionCohereTokenizerFastbark)bart)BartTokenizerBartTokenizerFastbarthezBarthezTokenizerBarthezTokenizerFast)bartpho)BartphoTokenizerNbertzbert-generationBertGenerationTokenizer)zbert-japanese)BertJapaneseTokenizerN)bertweet)BertweetTokenizerNbig_birdBigBirdTokenizerBigBirdTokenizerFastbigbird_pegasusPegasusTokenizerPegasusTokenizerFast)biogpt)BioGptTokenizerNbitnetr   )
blenderbot)BlenderbotTokenizerBlenderbotTokenizerFast)zblenderbot-small)BlenderbotSmallTokenizerNblipzblip-2GPT2TokenizerGPT2TokenizerFastbloomBloomTokenizerFastbridgetowerRobertaTokenizerRobertaTokenizerFastbros)byt5)ByT5TokenizerN	camembertCamembertTokenizerCamembertTokenizerFast)canine)CanineTokenizerN	chameleonchinese_clipclapclipCLIPTokenizerCLIPTokenizerFastclipseg)clvp)ClvpTokenizerN
code_llamaCodeLlamaTokenizerCodeLlamaTokenizerFastcodegenCodeGenTokenizerCodeGenTokenizerFastcoherecohere2colpaliconvbertConvBertTokenizerConvBertTokenizerFastcpmCpmTokenizerCpmTokenizerFast)cpmant)CpmAntTokenizerN)ctrl)CTRLTokenizerN)zdata2vec-audioWav2Vec2CTCTokenizerNzdata2vec-textdbrxdebertaDebertaTokenizerDebertaTokenizerFastz
deberta-v2DebertaV2TokenizerDebertaV2TokenizerFastdeepseek_v3	diffllama
distilbertDistilBertTokenizerDistilBertTokenizerFastdprDPRQuestionEncoderTokenizerDPRQuestionEncoderTokenizerFastelectraElectraTokenizerElectraTokenizerFastemu3ernieernie_mErnieMTokenizer)esm)EsmTokenizerNfalconfalcon_mambaGPTNeoXTokenizerFastfastspeech2_conformerFastSpeech2ConformerTokenizer)flaubert)FlaubertTokenizerNfnetFNetTokenizerFNetTokenizerFast)fsmt)FSMTTokenizerNfunnelFunnelTokenizerFunnelTokenizerFastgemmaGemmaTokenizerGemmaTokenizerFastgemma2gemma3gemma3_textgitglmglm4zgpt-sw3GPTSw3Tokenizergpt2gpt_bigcodegpt_neogpt_neox)gpt_neox_japanese)GPTNeoXJapaneseTokenizerNgptj)zgptsan-japanese)GPTSanJapaneseTokenizerNzgrounding-dinogroupvitheliumherbertHerbertTokenizerHerbertTokenizerFast)hubertrs   ibertideficsidefics2idefics3instructblipinstructblipvideointernvlQwen2TokenizerQwen2TokenizerFastjambajanusjetmoe)jukebox)JukeboxTokenizerNzkosmos-2XLMRobertaTokenizerXLMRobertaTokenizerFastlayoutlmLayoutLMTokenizerLayoutLMTokenizerFast
layoutlmv2LayoutLMv2TokenizerLayoutLMv2TokenizerFast
layoutlmv3LayoutLMv3TokenizerLayoutLMv3TokenizerFast	layoutxlmLayoutXLMTokenizerLayoutXLMTokenizerFastledLEDTokenizerLEDTokenizerFastliltllamallama4llama4_textllava
llava_nextllava_next_videollava_onevision
longformerLongformerTokenizerLongformerTokenizerFastlongt5T5TokenizerT5TokenizerFast)luke)LukeTokenizerNlxmertLxmertTokenizerLxmertTokenizerFastm2m_100M2M100Tokenizermambamamba2marianMarianTokenizermbartMBartTokenizerMBartTokenizerFastmbart50MBart50TokenizerMBart50TokenizerFastmegazmegatron-bert)zmgp-str)MgpstrTokenizerNmistralmixtralmllamamlukeMLukeTokenizer
mobilebertMobileBertTokenizerMobileBertTokenizerFast
modernbert	moonshinemoshimpnetMPNetTokenizerMPNetTokenizerFastmptmramt5MT5TokenizerMT5TokenizerFastmusicgenmusicgen_melodymvpMvpTokenizerMvpTokenizerFast)myt5)MyT5TokenizerNnemotronnezhanllbNllbTokenizerNllbTokenizerFastznllb-moenystromformerolmoolmo2olmoezomdet-turbo	oneformerz
openai-gptOpenAIGPTTokenizerOpenAIGPTTokenizerFastoptowlv2owlvit	paligemmapegasus	pegasus_x)	perceiver)PerceiverTokenizerN	persimmonphiphi3phimoe)phobert)PhobertTokenizerN
pix2structpixtralplbartPLBartTokenizer)
prophetnet)ProphetNetTokenizerNqdqbertqwen2qwen2_5_omni
qwen2_5_vlqwen2_audio	qwen2_moeqwen2_vlqwen3	qwen3_moe)rag)RagTokenizerNrealmRealmTokenizerRealmTokenizerFastrecurrent_gemmareformerReformerTokenizerReformerTokenizerFastrembertRemBertTokenizerRemBertTokenizerFast	retribertRetriBertTokenizerRetriBertTokenizerFastrobertazroberta-prelayernorm)roc_bert)RoCBertTokenizerNroformerRoFormerTokenizerRoFormerTokenizerFastrwkvseamless_m4tSeamlessM4TTokenizerSeamlessM4TTokenizerFastseamless_m4t_v2shieldgemma2siglipSiglipTokenizersiglip2speech_to_textSpeech2TextTokenizer)speech_to_text_2)Speech2Text2TokenizerNspeecht5SpeechT5Tokenizer)splinter)SplinterTokenizerSplinterTokenizerFastsqueezebertSqueezeBertTokenizerSqueezeBertTokenizerFaststablelm
starcoder2switch_transformerst5)tapas)TapasTokenizerN)tapex)TapexTokenizerN)z
transfo-xl)TransfoXLTokenizerNtvpudopUdopTokenizerUdopTokenizerFastumt5video_llavaviltvipllavavisual_bert)vits)VitsTokenizerN)wav2vec2rs   )zwav2vec2-bertrs   )zwav2vec2-conformerrs   )wav2vec2_phoneme)Wav2Vec2PhonemeCTCTokenizerNwhisperWhisperTokenizerWhisperTokenizerFastxclipxglmXGLMTokenizerXGLMTokenizerFast)xlm)XLMTokenizerNzxlm-prophetnetXLMProphetNetTokenizerzxlm-robertazxlm-roberta-xlxlnetXLNetTokenizerXLNetTokenizerFastxmodyosozambazamba2
class_namec                    U S:X  a  [         $ [        R                  5        H=  u  pX;   d  M  [        U5      n[        R
                  " SU 3S5      n [        X05      s  $    [        R                  R                  5        H%  u  pBU H  n[        USS 5      U :X  d  M  Us  s  $    M'     [        R
                  " S5      n[        X`5      (       a  [        X`5      $ g ! [         a     M  f = f)Nr   .ztransformers.models__name__transformers)r   r    itemsr   	importlibimport_modulegetattrAttributeErrorTOKENIZER_MAPPING_extra_contenthasattr)r  module_name
tokenizersmoduleconfig	tokenizermain_modules          b/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/auto/tokenization_auto.pytokenizer_class_from_namer    s    ..&&#:#@#@#B#3K@K,,q->@UVFv22 $C 0>>DDF#Iy*d3zA   $ G )).9K{''{// " s   
C
C)(C)pretrained_model_name_or_path	cache_dirforce_downloadresume_downloadproxiestokenrevisionlocal_files_only	subfolderc	                    U	R                  SS5      n
U
b+  [        R                  " S[        5        Ub  [	        S5      eU
nU	R                  SS5      n[        U [        UUUUUUUUSSSUS9nUc  [        R                  S5        0 $ [        X5      n[        US	S
9 n[        R                  " U5      nSSS5        UWS'   U$ ! , (       d  f       N= f)aE  
Loads the tokenizer configuration from a pretrained model tokenizer configuration.

Args:
    pretrained_model_name_or_path (`str` or `os.PathLike`):
        This can be either:

        - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
          huggingface.co.
        - a path to a *directory* containing a configuration file saved using the
          [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

    cache_dir (`str` or `os.PathLike`, *optional*):
        Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
        cache should not be used.
    force_download (`bool`, *optional*, defaults to `False`):
        Whether or not to force to (re-)download the configuration files and override the cached versions if they
        exist.
    resume_download:
        Deprecated and ignored. All downloads are now resumed by default when possible.
        Will be removed in v5 of Transformers.
    proxies (`Dict[str, str]`, *optional*):
        A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
        'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
    token (`str` or *bool*, *optional*):
        The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
        when running `huggingface-cli login` (stored in `~/.huggingface`).
    revision (`str`, *optional*, defaults to `"main"`):
        The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
        git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
        identifier allowed by git.
    local_files_only (`bool`, *optional*, defaults to `False`):
        If `True`, will only try to load the tokenizer configuration from local files.
    subfolder (`str`, *optional*, defaults to `""`):
        In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can
        specify the folder name here.

<Tip>

Passing `token=True` is required when you want to use a private model.

</Tip>

Returns:
    `Dict`: The configuration of the tokenizer.

Examples:

```python
# Download configuration from huggingface.co and cache.
tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased")
# This model does not have a tokenizer config so the result will be an empty dict.
tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base")

# Save a pretrained tokenizer locally and you can reload its config
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
tokenizer.save_pretrained("tokenizer-test")
tokenizer_config = get_tokenizer_config("tokenizer-test")
```use_auth_tokenNrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`._commit_hashF)r  r  r  r  r  r  r  r   _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsr  z\Could not locate the tokenizer configuration file, will try to use the model config instead.zutf-8)encoding)popwarningswarnFutureWarning
ValueErrorgetr   r   loggerinfor   openjsonload)r  r  r  r  r  r  r  r  r  kwargsr  commit_hashresolved_config_filereaderresults                  r  get_tokenizer_configr    s    R ZZ 0$7N! A	
 uvv**^T2K&%%'))..305   #rs	%&:HK	"W	56" 
6(F>M 
6	5s   C
Cc                   X    \ rS rSrSrS r\\" \5      S 5       5       r	\
SS j5       rSrg)	AutoTokenizeri)  a  
This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
created with the [`AutoTokenizer.from_pretrained`] class method.

This class cannot be instantiated directly using `__init__()` (throws an error).
c                     [        S5      e)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)EnvironmentError)selfs    r  __init__AutoTokenizer.__init__1  s    _
 	
    c           
      P
   UR                  SS5      nUb=  [        R                  " S[        5        UR	                  SS5      b  [        S5      eXCS'   UR                  SS5      nSUS'   UR                  S	S5      nUR                  S
S5      nUR                  SS5      nUR	                  SS5      n	Ub  Sn
[        R	                  US5      nUc:  [        SU SSR                  S [        R                  5        5       5       S35      eUu  pU(       a$  Ub  [        U5      n
O[        R                  S5        U
c  [        U5      n
U
c  [        SU S35      eU
R                  " U/UQ70 UD6$ [        U40 UD6nSU;   a  US   US'   UR	                  S5      nSnSU;   a9  [        US   [        [         45      (       a  US   nOUS   R	                  SS5      nUc  [        U["        5      (       dP  U	(       a0  [%        X40 UD6n['        USS9S   n[(        R*                  " S'0 UD6nO[(        R                  " U4SU0UD6nUR,                  n[/        US5      (       a  SUR0                  ;   a  UR0                  S   nUSLn[3        U5      [4        ;   =(       d/    USL=(       a$    [        U5      SL=(       d    [        US-   5      SLn[7        XUU5      nU(       a  U(       a  U(       a  US   b  US   nOUS   n[9        UU40 UD6n
UR                  SS5      n[:        R<                  R?                  U5      (       a  U
RA                  5         U
R                  " U/UQ7SU0UD6$ Ubg  Sn
U(       a&  URC                  S5      (       d  U S3n[        U5      n
U
c  Un[        U5      n
U
c  [        SW S35      eU
R                  " U/UQ70 UD6$ [        U[D        5      (       a{  [3        URF                  5      [3        URH                  5      LaD  [        R                  S URH                  RJ                   S!URF                  RJ                   S"35        URH                  n[M        [3        U5      RN                  5      nUb`  [4        [3        U5         u  nnU(       a   U(       d  Uc  UR                  " U/UQ70 UD6$ Ub  UR                  " U/UQ70 UD6$ [        S#5      e[        S$URJ                   S%SR                  S& [4        R                  5        5       5       S35      e)(ae  
Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.

The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
falling back to using pattern matching on `pretrained_model_name_or_path`:

List options

Params:
    pretrained_model_name_or_path (`str` or `os.PathLike`):
        Can be either:

            - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
            - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
              using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
            - A path or url to a single saved vocabulary file if and only if the tokenizer only requires a
              single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
              applicable to all derived classes)
    inputs (additional positional arguments, *optional*):
        Will be passed along to the Tokenizer `__init__()` method.
    config ([`PretrainedConfig`], *optional*)
        The configuration object used to determine the tokenizer class to instantiate.
    cache_dir (`str` or `os.PathLike`, *optional*):
        Path to a directory in which a downloaded pretrained model configuration should be cached if the
        standard cache should not be used.
    force_download (`bool`, *optional*, defaults to `False`):
        Whether or not to force the (re-)download the model weights and configuration files and override the
        cached versions if they exist.
    resume_download:
        Deprecated and ignored. All downloads are now resumed by default when possible.
        Will be removed in v5 of Transformers.
    proxies (`Dict[str, str]`, *optional*):
        A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
        'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
    revision (`str`, *optional*, defaults to `"main"`):
        The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
        git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
        identifier allowed by git.
    subfolder (`str`, *optional*):
        In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for
        facebook/rag-token-base), specify it here.
    use_fast (`bool`, *optional*, defaults to `True`):
        Use a [fast Rust-based tokenizer](https://huggingface.co/docs/tokenizers/index) if it is supported for
        a given model. If a fast tokenizer is not available for a given model, a normal Python-based tokenizer
        is returned instead.
    tokenizer_type (`str`, *optional*):
        Tokenizer type to be loaded.
    trust_remote_code (`bool`, *optional*, defaults to `False`):
        Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
        should only be set to `True` for repositories you trust and in which you have read the code, as it will
        execute code present on the Hub on your local machine.
    kwargs (additional keyword arguments, *optional*):
        Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
        `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
        `additional_special_tokens`. See parameters in the `__init__()` for more details.

Examples:

```python
>>> from transformers import AutoTokenizer

>>> # Download vocabulary from huggingface.co and cache.
>>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

>>> # Download vocabulary from huggingface.co (user-uploaded) and cache.
>>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased")

>>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*)
>>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/")

>>> # Download vocabulary from huggingface.co and define model-specific arguments
>>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True)
```r  Nr  r  r  r  T
_from_autouse_fasttokenizer_typetrust_remote_code	gguf_filezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c              3   $   #    U  H  ov   M     g 7fN .0cs     r  	<genexpr>0AutoTokenizer.from_pretrained.<locals>.<genexpr>  s      K,Jq,Js   r  zt`use_fast` is set to `True` but the tokenizer class does not have a fast version.  Falling back to the slow version.zTokenizer class z is not currently imported.r  tokenizer_classauto_mapr  F)return_tensorsFastr   r   code_revisionz- does not exist or is not currently imported.z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.zzThis tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer.z!Unrecognized configuration class z8 to build an AutoTokenizer.
Model type should be one of c              3   8   #    U  H  oR                   v   M     g 7fr  )r  r  s     r  r  r    s     4bIaAZZIas   r  )(r  r  r  r  r  r  r    joinkeysr  r  warningfrom_pretrainedr  
isinstancetuplelistr
   r   r   r   	for_modelr  r  r  typer  r   r   ospathisdirregister_for_auto_classendswithr   decoderencoder	__class__r   r  )clsr  inputsr  r  r  r  r  r  r  r  tokenizer_class_tupletokenizer_class_nametokenizer_fast_class_nametokenizer_configconfig_tokenizer_classtokenizer_auto_map	gguf_pathconfig_dicthas_remote_codehas_local_code	class_ref_tokenizer_class_candidate
model_typetokenizer_class_pytokenizer_class_fasts                              r  r  AutoTokenizer.from_pretrained7  s   Z  $4d;%MM E zz'4(4 l  -7OHd+#|::j$/$4d;"JJ':DAJJ{D1	 %"O$;$?$?PT$U!$, .~.>>qyy K,C,H,H,J KKLAO 
 ?T; ,8&?@Y&ZONN= &";<P"Q& #34H3IId!eff"223PdSYd]cdd 00MXQWX--%5n%EF>"!1!5!56G!H!))*:6FF%5j%A"%5j%A%E%EoW[%\" ")f&677 +,I _X^ _I"6yQV"WX`"aK'11@K@F'775IZ^dF &,%;%;"vz**&///Q%+___%E",D8f):: 
"$. )*@AM Z,-Cf-LMUYY	 	 6no
 0.q1=.q1	.q1	;IGdohnoO

?D1Aww}}:;;779"22-06J[_e  $/"O 6 ? ? G G/E.Fd,K)";<U"V&,B)";<U"V& &'@&AAno  #223PdSYd]cdd f233FNN#4+??6v~~7O7O6P Q%%+^^%=%=$> ?22 ^^F/V0E0EF
!7Hf7V4 4#5G5O+;;<Ym\bmflmm%1-==>[o^dohnoo$: 
 /0@0@/A B++/994bIZI_I_Ia4b+b*ccdf
 	
r  Nc                    Uc  Uc  [        S5      eUb   [        U[        5      (       a  [        S5      eUb   [        U[        5      (       a  [        S5      eUbD  UbA  [        U[        5      (       a,  UR                  U:w  a  [        SUR                   SU S35      eU [
        R                  ;   a  [
        U    u  pEUc  UnUc  Un[
        R                  XU4US9  g)	ar  
Register a new tokenizer in this mapping.


Args:
    config_class ([`PretrainedConfig`]):
        The configuration corresponding to the model to register.
    slow_tokenizer_class ([`PretrainedTokenizer`], *optional*):
        The slow tokenizer to register.
    fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*):
        The fast tokenizer to register.
NzKYou need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_classz:You passed a fast tokenizer in the `slow_tokenizer_class`.z:You passed a slow tokenizer in the `fast_tokenizer_class`.zThe fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not consistent with the slow tokenizer class you passed (fast tokenizer has z and you passed z!. Fix one of those so they match!)exist_ok)r  
issubclassr   r   slow_tokenizer_classr  r  register)config_classr  fast_tokenizer_classr  existing_slowexisting_fasts         r  r  AutoTokenizer.register  s     ',@,Hjkk+
;OQh0i0iYZZ+
;OQd0e0eYZZ !,$0/1HII$99=QQ['<<==MNbMc d!!  ,;;;+<\+J(M#+'4$#+'4$""<H\1]hp"qr  r  )NNF)r  
__module____qualname____firstlineno____doc__r  classmethodr   r    r  staticmethodr  __static_attributes__r  r  r  r  r  )  sH    
 &'>?\
 @ \
| )r )rr  r  r  )NFNNNNF ):r  r  r  r  r  collectionsr   typingr   r   r   r   r   configuration_utilsr
   dynamic_module_utilsr   r   modeling_gguf_pytorch_utilsr   tokenization_utilsr   tokenization_utils_baser   utilsr   r   r   r   r   r   encoder_decoderr   auto_factoryr   configuration_autor   r   r   r   r   tokenization_utils_fastr   
get_loggerr  r  r    str__annotations__r  r  CONFIG_TO_TYPEr  PathLikeboolr  r  __all__)kvs   00r  <module>r'     s%!      	  # > > 3 \ ? 5 <  3 *  B" 
		H	% VaUb[eHSM8C=4P.Q)QRb)X		
)C)E)E%4-D-F-F)DX		
 ?V?X?X(;^bcdX		
 &@W@Y@Y(<_cdeX		
 D;R;T;T"7Z^_`X		
 o>U>W>W':]abcX		
 =X		
 *D*F*F&D.E.G.G*TX		
( 4)X		
* o>U>W>W':]abc+X		
, >X>Z>Z!:`dfj kl-X		
. ?/X		
0 61X		
4 *D*F*F&D.E.G.G*T3X		
@ !3OfOhOh5Knr stAX		
B 2CX		
D ;R;T;T7Z^_`EX		
F OGX		
H EIX		
J o>U>W>W':]abcKX		
L @W@Y@Y)<_cdeMX		
N t5L5N5N1TXYZOX		
P /KbKdKd1GjnopQX		
R o>U>W>W':]abcSX		
T .UX		
X ,F,H,H(d0G0I0I,tWX		
d 2eX		
h (B(D(D$$,C,E,E(4gX		
t oF]F_F_/BeijkuX		
x &.E.G.G*TwX		
F #+B+D+D'$EX		
T #+B+D+D'$SX		
` .aX		
d ,F,H,H(d0G0I0I,tcX		
p +G^G`G`-CfjklqX		
r 7N7P7P3VZ[\sX		
t 8O8Q8Q4W[\]uX		
v )CZC\C\+?bfghwX		
x -JaJcJc/FimnoyX		
| &@&B&BN*A*C*C&{X		
H 2IX		
J .KX		
L ?MX		
N 1MdMfMf3IlpqrOX		
P o>U>W>W':]abcQX		
R +G^G`G`-CfjklSX		
V ,F,H,H(d0G0I0I,tUX		
d (B(D(D$$,C,E,E(4cX		
r (B(D(D$$,C,E,E(4qX		
~ 1PgPiPi3LostuX		
B 19P9R9R5X\AX		
N +G^G`G`-CfjklOX		
P o>U>W>W':]abcQX		
R ?V?X?X(;^bcdSX		
T .H.J.J*PTVZ[\UX		
V ,WX		
X ;R;T;T7Z^_`YX		
Z d>U>W>W$:]abc[X		
^ (4G4I4I0tUYZ]X		
d 6eX		
f o>U>W>W':]abcgX		
h .iX		
j )D[D]D]+@cghikX		
n (B(D(D$$,C,E,E(4mX		
| (B(D(D$$,C,E,E(4{X		
J (B(D(D$$,C,E,E(4IX		
X (B(D(D$$,C,E,E(4WX		
d _=T=V=V&9\`abeX		
f T8O8Q8Q4W[\]gX		
h d9P9R9R5X\]^iX		
j .H.J.J*PTVZ[\kX		
l o>U>W>W':]abcmX		
n _E\E^E^.AdhijoX		
p AXAZAZ*=`defqX		
r $:Q:S:S 6Y]^_sX		
t FuX		
v o>U>W>W':]abcwX		
x CyX		
z H_HaHa1Dgklm{X		
| /BYB[B[+>aefg}X		
~ ;R;T;T7Z^_`X		
@ +G^G`G`-CfjklAX		
B 7CX		
D )E\E^E^+AdhijEX		
F 7N7P7P3VZ[\GX		
H *D[D]D],@cghiIX		
J *D[D]D],@cghiKX		
L oF]F_F_/BeijkMX		
N !?KbKdKd4Gjn"opOX		
P *D[D]D],@cghiQX		
T (B(D(D$$,C,E,E(4SX		
` t5L5N5N1TXYZaX		
d (B(D(D$$,C,E,E(4cX		
p 4qX		
t -G-I-I)t1H1J1J-PTsX		
@ -JaJcJc/FimnoAX		
B 1PgPiPi3LostuCX		
D 1PgPiPi3LostuEX		
F /MdMfMf1IlpqrGX		
H ^;R;T;T%7Z^_`IX		
J +JaJcJc-FimnoKX		
N (B(D(D$$,C,E,E(4MX		
\ (B(D(D$$,C,E,E(4[X		
j (B(D(D$$,C,E,E(4iX		
v 'AXAZAZ)=`defwX		
x ,F]F_F_.BeijkyX		
z  "2LcLeLe4Hko!pq{X		
| !1KbKdKd3Gjn op}X		
~ 1PgPiPi3LostuX		
B %?%A%AMt)@)B)B%AX		
N .OX		
P )D[D]D]+@cghiQX		
R .H.J.J*PTVZ[\SX		
T t7N7P7P3VZ[\UX		
V 8O8Q8Q4W[\]WX		
X -G-I-I)tUYZ[YX		
\ (B(D(D$$,C,E,E(4[X		
j *D*F*F&D.E.G.G*TiX		
v (D[D]D]*@cghiwX		
x G^G`G`0CfjklyX		
z 3{X		
~ (B(D(D$$,C,E,E(4}X		
L	 (B(D(D$$,C,E,E(4K	X		
X	 (BYB[B[*>aefgY	X		
Z	 +E+G+G'TSWXY[	X		
\	 1PgPiPi3Lostu]	X		
^	 D?V?X?X";^bcd_	X		
`	 4>U>W>W!:]abca	X		
b	 t:Q:S:S6Y]^_c	X		
d	 'AXAZAZ)=`defe	X		
f	 T5L5N5N1TXYZg	X		
h	 'CZC\C\)?bfghi	X		
l	 &@&B&BN*A*C*C&k	X		
x	 ->U>W>W):]abcy	X		
z	 E\E^E^0Adh ij{	X		
|	 ^;R;T;T%7Z^_`}	X		
~	 .	X		
@
 $=T=V=V 9\`abA
X		
B
 ?V?X?X(;^bcdC
X		
F
 'A'C'CO+B+D+D'$E
X		
T
 'A'C'CO+B+D+D'$S
X		
b
  )C)E)E%4-D-F-F)Da
X		
n
 d6M6O6O2UYZ[o
X		
p
 t7N7P7P3VZ[\q
X		
r
 t7N7P7P3VZ[\s
X		
v
  9P9R9R"5X\]u
X		
|
 ?CZC\C\,?bfgh}
X		
@ %CZC\C\'?bfg
X		
F _=T=V=V&9\`abGX		
H ?V?X?X(;^bcdIX		
J @W@Y@Y)<_cdeKX		
L +E\E^E^-AdhijMX		
P *D*F*F&D.E.G.G*TOX		
^ *D*F*F&D.E.G.G*T]X		
jkX		
z (B(D(D$$,C,E,E(4yX		
F 'CZC\C\)?bfghGX		
H &@W@Y@Y(<_cdeIX		
J (BYB[B[*>aefgKX		
L 4MX		
N M@W@Y@Y+<_cdeOX		
P <S<U<U8[_`aQX		
R -G-I-I)tUYZ[SX		
T :UX		
V AXAZAZ*=`defWX		
Z $,C,E,E(4YX		
f .H_HaHa0DgklmgX		
h ,F]F_F_.BeijkiX		
j -G^G`G`/CfjklkX		
n $,C,E,E(4mX		
z *D[D]D],@cghi{X		
~ $,C,E,E(4}X		
L $,C,E,E(4KX		
X ,YX		
Z 'AXAZAZ)=`def[X		
^ "(B(D(D$$,C,E,E(4]X		
l +E+G+G'T/F/H/H+dkX		
z *D*F*F&D.E.G.G*TyX		
F /MdMfMf1IlpqrGX		
H +G^G`G`-CfjklIX		
L '#?V?X?X%;^bcKX		
R 5SX		
T -JaJcJc/FimnoUX		
V d6M6O6O2UYZ[WX		
Z .H.J.J*PT2I2K2K.QUYX		
h ".H.J.J*PT2I2K2K.QUgX		
v (B(D(D$$,C,E,E(4uX		
B -G-I-I)tUYZ[CX		
F (B(D(D$$,C,E,E(4EX		
R :T:V:V 6\`bfghSX		
T BUX		
V 1K1M1M-SWY]^_WX		
X IYX		
\ 'G^G`G`)Cfjk[X		
b $:Q:S:S 6Y]^_cX		
d OD[D]D]-@cghieX		
h &%?%A%AMt)@)B)B%gX		
v %?%A%AMt)@)B)B%uX		
B 0CX		
D 0EX		
F 9GX		
H _=T=V=V&9\`abIX		
L 'A'C'CO+B+D+D'$KX		
Z %?%A%AMt)@)B)B%YX		
f -G^G`G`/CfjklgX		
h o>U>W>W':]abciX		
j *D[D]D],@cghikX		
l _E\E^E^.AdhijmX		
n .oX		
p 9qX		
r >sX		
t CuX		
v HwX		
x +G^G`G`-CfjklyX		
z ?V?X?X(;^bcd{X		
~ 'A'C'CO+B+D+D'$}X		
J ,KX		
L <V<X<X 8^bdhijMX		
P -G-I-I)t1H1J1J-PTOX		
^ !-G-I-I)t1H1J1J-PT]X		
l (B(D(D$$,C,E,E(4kX		
z -G-I-I)t1H1J1J-PTyX		
H )C)E)E%4-D-F-F)DGX		
V (B(D(D$$,C,E,E(4UX		
d (B(D(D$$,C,E,E(4cX		
Z	x %%9;RS #7#=#=#?@#?41!$#?@# < 48 &*(,(,""l#(bkk)9#:lc2;;./0l l d^	l
 d38n%l E$)$%l sml l l^Xr Xrv 
0S As   KAM-