
    fTh                         S r SSKrSSKrSSKJrJr  SSKJr  SSKJ	r	  SSK
Jr  \	R                  " \5      r " S	 S
5      rS
/rg)zTokenization classes for RAG.    N)ListOptional   )BatchEncoding)logging   )	RagConfigc                       \ rS rSrS rS r\S 5       rS rS r	S r
S rS	 r      SS\\   S\\\      S\\   S\\   S\S\\   S\S\4S jjrSrg
)RagTokenizer   c                 >    Xl         X l        U R                   U l        g N)question_encoder	generatorcurrent_tokenizer)selfr   r   s      `/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/rag/tokenization_rag.py__init__RagTokenizer.__init__   s     0"!%!6!6    c                    [         R                  R                  U5      (       a  [        SU S35      e[         R                  " USS9  [         R                  R                  US5      n[         R                  R                  US5      nU R                  R                  U5        U R                  R                  U5        g )NzProvided path (z#) should be a directory, not a fileT)exist_okquestion_encoder_tokenizergenerator_tokenizer)	ospathisfile
ValueErrormakedirsjoinr   save_pretrainedr   )r   save_directoryquestion_encoder_pathgenerator_paths       r   r!   RagTokenizer.save_pretrained#   s    77>>.))~.>>abcc
NT2 "^=Y Zn6KL--.CD&&~6r   c                     SSK Jn  UR                  SS 5      nUc  [        R                  " U5      nUR	                  XR
                  SS9nUR	                  XR                  SS9nU " XVS9$ )N   )AutoTokenizerconfigr   )r)   	subfolderr   )r   r   )auto.tokenization_autor(   popr	   from_pretrainedr   r   )clspretrained_model_name_or_pathkwargsr(   r)   r   r   s          r   r-   RagTokenizer.from_pretrained,   s     	;Hd+>../LMF(88)2I2IUq 9 
 "11)2B2BNc 2 
	 $4JJr   c                 &    U R                   " U0 UD6$ r   )r   r   argsr0   s      r   __call__RagTokenizer.__call__>   s    %%t6v66r   c                 :    U R                   R                  " U0 UD6$ r   )r   batch_decoder3   s      r   r8   RagTokenizer.batch_decodeA   s    ~~**D;F;;r   c                 :    U R                   R                  " U0 UD6$ r   )r   decoder3   s      r   r;   RagTokenizer.decodeD   s    ~~$$d5f55r   c                 &    U R                   U l        g r   )r   r   r   s    r   _switch_to_input_mode"RagTokenizer._switch_to_input_modeG   s    !%!6!6r   c                 &    U R                   U l        g r   )r   r   r>   s    r   _switch_to_target_mode#RagTokenizer._switch_to_target_modeJ   s    !%r   N	src_texts	tgt_texts
max_lengthmax_target_lengthpaddingreturn_tensors
truncationreturnc           
          [         R                  " S[        5        Uc  U R                  R                  nU " U4SUUUUS.UD6n	Uc  U	$ Uc  U R                  R                  nU " SUSUUUUS.UD6n
U
S   U	S'   U	$ )Nu4  `prepare_seq2seq_batch` is deprecated and will be removed in version 5 of 🤗 Transformers. Use the regular `__call__` method to prepare your inputs and the tokenizer under the `with_target_tokenizer` context manager to prepare your targets. See the documentation of your specific tokenizer for more detailsT)add_special_tokensrI   rF   rH   rJ   )text_targetrM   rI   rH   rF   rJ   	input_idslabels )warningswarnFutureWarningr   model_max_length)r   rD   rE   rF   rG   rH   rI   rJ   r0   model_inputsrP   s              r   prepare_seq2seq_batch"RagTokenizer.prepare_seq2seq_batchM   s     	 	
 //@@J
#)!!
 
 $ $ 6 6 G G 
!#)(!
 
 "(!4Xr   )r   r   r   )NNNlongestNT)__name__
__module____qualname____firstlineno__r   r!   classmethodr-   r5   r8   r;   r?   rB   r   strr   intboolr   rW   __static_attributes__rQ   r   r   r   r      s    7
7 K K"7<670 *.$(+/ (,,9, DI&, SM	,
 $C=, , !, , 
, ,r   r   )__doc__r   rR   typingr   r   tokenization_utils_baser   utilsr   configuration_ragr	   
get_loggerrZ   loggerr   __all__rQ   r   r   <module>rk      sE    $ 	  ! 4  ( 
		H	%\ \~ 
r   