
    fTh7                        S SK JrJr   " S S\S9r " S S\S9r " S S\S9r " S	 S
\S9r " S S\S9r " S S\S9r " S S\S9r	 " S S\S9r
 " S S\S9r " S S\S9r " S S\S9r " S S\S9r " S S\S9r " S S\S9r " S S \S9r " S! S"\S9r " S# S$\S9r " S% S&\S9r " S' S(\S9r " S) S*\S9r " S+ S,\S9r " S- S.\S9r " S/ S0\S9r " S1 S2\S9r " S3 S4\S9r " S5 S6\S9r " S7 S8\S9r " S9 S:\S9r " S; S<\S9r " S= S>\S9r  " S? S@\S9r! " SA SB\S9r" " SC SD\S9r# " SE SF\S9r$ " SG SH\S9r% " SI SJ\S9r&gK)L   )DummyObjectrequires_backendsc                        \ rS rSrS/rS rSrg)AlbertTokenizer   sentencepiecec                     [        U S/5        g Nr   r   selfargskwargss      f/var/www/auris/envauris/lib/python3.13/site-packages/transformers/utils/dummy_sentencepiece_objects.py__init__AlbertTokenizer.__init__       $ 12     N__name__
__module____qualname____firstlineno__	_backendsr   __static_attributes__r   r   r   r   r           !I3r   r   )	metaclassc                        \ rS rSrS/rS rSrg)BarthezTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   BarthezTokenizer.__init__   r   r   r   Nr   r   r   r   r    r       r   r   r    c                        \ rS rSrS/rS rSrg)BartphoTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   BartphoTokenizer.__init__   r   r   r   Nr   r   r   r   r%   r%      r   r   r%   c                        \ rS rSrS/rS rSrg)BertGenerationTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r    BertGenerationTokenizer.__init__   r   r   r   Nr   r   r   r   r*   r*      r   r   r*   c                        \ rS rSrS/rS rSrg)BigBirdTokenizer!   r   c                     [        U S/5        g r
   r   r   s      r   r   BigBirdTokenizer.__init__$   r   r   r   Nr   r   r   r   r/   r/   !   r   r   r/   c                        \ rS rSrS/rS rSrg)CamembertTokenizer(   r   c                     [        U S/5        g r
   r   r   s      r   r   CamembertTokenizer.__init__+   r   r   r   Nr   r   r   r   r4   r4   (   r   r   r4   c                        \ rS rSrS/rS rSrg)CodeLlamaTokenizer/   r   c                     [        U S/5        g r
   r   r   s      r   r   CodeLlamaTokenizer.__init__2   r   r   r   Nr   r   r   r   r9   r9   /   r   r   r9   c                        \ rS rSrS/rS rSrg)CpmTokenizer6   r   c                     [        U S/5        g r
   r   r   s      r   r   CpmTokenizer.__init__9   r   r   r   Nr   r   r   r   r>   r>   6   r   r   r>   c                        \ rS rSrS/rS rSrg)DebertaV2Tokenizer=   r   c                     [        U S/5        g r
   r   r   s      r   r   DebertaV2Tokenizer.__init__@   r   r   r   Nr   r   r   r   rC   rC   =   r   r   rC   c                        \ rS rSrS/rS rSrg)ErnieMTokenizerD   r   c                     [        U S/5        g r
   r   r   s      r   r   ErnieMTokenizer.__init__G   r   r   r   Nr   r   r   r   rH   rH   D   r   r   rH   c                        \ rS rSrS/rS rSrg)XLMProphetNetTokenizerK   r   c                     [        U S/5        g r
   r   r   s      r   r   XLMProphetNetTokenizer.__init__N   r   r   r   Nr   r   r   r   rM   rM   K   r   r   rM   c                        \ rS rSrS/rS rSrg)FNetTokenizerR   r   c                     [        U S/5        g r
   r   r   s      r   r   FNetTokenizer.__init__U   r   r   r   Nr   r   r   r   rR   rR   R   r   r   rR   c                        \ rS rSrS/rS rSrg)GemmaTokenizerY   r   c                     [        U S/5        g r
   r   r   s      r   r   GemmaTokenizer.__init__\   r   r   r   Nr   r   r   r   rW   rW   Y   r   r   rW   c                        \ rS rSrS/rS rSrg)GPTSw3Tokenizer`   r   c                     [        U S/5        g r
   r   r   s      r   r   GPTSw3Tokenizer.__init__c   r   r   r   Nr   r   r   r   r\   r\   `   r   r   r\   c                        \ rS rSrS/rS rSrg)LayoutXLMTokenizerg   r   c                     [        U S/5        g r
   r   r   s      r   r   LayoutXLMTokenizer.__init__j   r   r   r   Nr   r   r   r   ra   ra   g   r   r   ra   c                        \ rS rSrS/rS rSrg)LlamaTokenizern   r   c                     [        U S/5        g r
   r   r   s      r   r   LlamaTokenizer.__init__q   r   r   r   Nr   r   r   r   rf   rf   n   r   r   rf   c                        \ rS rSrS/rS rSrg)M2M100Tokenizeru   r   c                     [        U S/5        g r
   r   r   s      r   r   M2M100Tokenizer.__init__x   r   r   r   Nr   r   r   r   rk   rk   u   r   r   rk   c                        \ rS rSrS/rS rSrg)MarianTokenizer|   r   c                     [        U S/5        g r
   r   r   s      r   r   MarianTokenizer.__init__   r   r   r   Nr   r   r   r   rp   rp   |   r   r   rp   c                        \ rS rSrS/rS rSrg)MBartTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   MBartTokenizer.__init__   r   r   r   Nr   r   r   r   ru   ru      r   r   ru   c                        \ rS rSrS/rS rSrg)MBart50Tokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   MBart50Tokenizer.__init__   r   r   r   Nr   r   r   r   rz   rz      r   r   rz   c                        \ rS rSrS/rS rSrg)MLukeTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   MLukeTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)MT5Tokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   MT5Tokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)NllbTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   NllbTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)PegasusTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   PegasusTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)PLBartTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   PLBartTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)ReformerTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   ReformerTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)RemBertTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   RemBertTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)SeamlessM4TTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   SeamlessM4TTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)SiglipTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   SiglipTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)Speech2TextTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   Speech2TextTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)SpeechT5Tokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   SpeechT5Tokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)T5Tokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   T5Tokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)UdopTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   UdopTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)XGLMTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   XGLMTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)XLMRobertaTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   XLMRobertaTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   c                        \ rS rSrS/rS rSrg)XLNetTokenizer   r   c                     [        U S/5        g r
   r   r   s      r   r   XLNetTokenizer.__init__   r   r   r   Nr   r   r   r   r   r      r   r   r   N)'utilsr   r   r   r    r%   r*   r/   r4   r9   r>   rC   rH   rM   rR   rW   r\   ra   rf   rk   rp   ru   rz   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      s   23 33 33 33 33 33; 33; 33[ 33; 33 33{ 33k 33{ 33 33; 33{ 33 33 33{ 33 33{ 33[ 33k 33 33 33+ 33 33[ 33 33[ 33+ 33K 33k 33k 33K 33{ 3r   