
    fTh"%                        S r SSKJr  SSKJrJrJrJr  SSKrSSKJ	r	  SSK
Jr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  \" SS9\ " S S\5      5       5       r\ " S S\5      5       r\" SS9 " S S\5      5       rSS/rg)zPyTorch ColPali model    )	dataclass)ListOptionalTupleUnionN)nn)AutoModelForImageTextToText   )Cache)PreTrainedModel)ModelOutputauto_docstring   )ColPaliConfigz_
    The bare ColPali model outputting raw hidden-states without any specific head on top.
    )custom_introc                   &    \ rS rSr\rSr/ rS rSr	g)ColPaliPreTrainedModel   modelc                    [        U R                  S5      (       a  U R                  R                  O)U R                  R                  R                  R                  n[        U[        R                  [        R                  45      (       aW  UR                  R                  R                  SUS9  UR                  b%  UR                  R                  R                  5         g g [        U[        R                  5      (       ad  UR                  R                  R                  SUS9  UR                  b2  UR                  R                  UR                     R                  5         g g g )Ninitializer_rangeg        )meanstd)hasattrconfigr   
vlm_configtext_config
isinstancer   LinearConv2dweightdatanormal_biaszero_	Embeddingpadding_idx)selfmoduler   s      d/var/www/auris/envauris/lib/python3.13/site-packages/transformers/models/colpali/modeling_colpali.py_init_weights$ColPaliPreTrainedModel._init_weights*   s    t{{$788 KK))''33EE 	 fryy"))455MM&&CS&9{{&  &&( '--MM&&CS&9!!-""6#5#56<<> . .     N)
__name__
__module____qualname____firstlineno__r   config_classbase_model_prefix_no_split_modulesr+   __static_attributes__r.   r-   r*   r   r      s     !L?r-   r   c                   &   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\\R                     \4      \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   Sr\\R                     \	S	'   S
rg)ColPaliForRetrievalOutput;   a  
Base class for ColPali embeddings output.

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The embeddings of the model.
    past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
Nloss
embeddingspast_key_valueshidden_states
attentionsimage_hidden_statesr.   )r/   r0   r1   r2   __doc__r:   r   torchFloatTensor__annotations__r;   Tensorr<   r   r   r   r=   r   r>   r?   r6   r.   r-   r*   r8   r8   ;   s    < )-D(5$$
%,)-J&-GKOXeD):):$;U$BCDK8<M8E%"3"345<59Ju001297;%"3"34;r-   r8   u\  
    In our proposed ColPali approach, we leverage VLMs to construct efficient multi-vector embeddings directly
    from document images (“screenshots”) for document retrieval. We train the model to maximize the similarity
    between these document embeddings and the corresponding query embeddings, using the late interaction method
    introduced in ColBERT.

    Using ColPali removes the need for potentially complex and brittle layout recognition and OCR pipelines with a
    single model that can take into account both the textual and visual content (layout, charts, etc.) of a document.
    c                   H  ^  \ rS rSrS\4U 4S jjr\      SS\\R                     S\\R                     S\\R                     S\\   S\\   S	\\   S
\\\4   4S jj5       rS rS rS rS rS rS rS r   SS\\   S\\   S\S
\R4                  4S jjrSrU =r$ )ColPaliForRetrievalc   r   c                 &  > [         TU ]  U5        Xl        UR                  R                  R
                  U l        [        R                  " UR                  5      nUR                  b%  UR                   Vs/ s H  nSU 3PM
     snU l        X l	        U R                  R                  U l
        [        R                  " U R                  R                  R                  R                  U R                  5      U l        U R                  5         g s  snf )Nzvlm.)super__init__r   r   r   
vocab_sizer	   from_config_tied_weights_keysvlmembedding_dimr   r   hidden_sizeembedding_proj_layer	post_init)r(   r   rN   k	__class__s       r*   rJ   ColPaliForRetrieval.__init__o   s      ++77BB)55f6G6GH!!-;>;Q;Q&R;QaaSz;Q&RD#![[66$&IIKK""..::%
!
 	 'Ss   7D	input_idspixel_valuesattention_maskoutput_attentionsoutput_hidden_statesreturn_dictreturnc           
         SU;   a  US   R                  U R                  S9US'   Ub  UOU R                  R                  nUb  UOU R                  R                  nUb  UOU R                  R
                  nU R                  " S	UUUSUUS.UD6nUR                  S   n	U R                  U	5      n
XR                  SSS9-  n
XR                  S5      -  n
S nU(       d6  U
4USS  -   nUb  US   OS US'   Ub  UR                  OS 4US'   Ub  U4U-   $ U$ [        UU
UR                  U(       a  UR                  OS UR                  Ub  UR                  S9$ S S9$ )
NrW   )dtypeT)rV   rX   rW   rZ   r[   rY   )dimkeepdim   )r:   r;   r<   r=   r>   r?   r.   )tor^   r   rY   rZ   use_return_dictrN   r=   rQ   norm	unsqueezer?   r8   r<   r>   )r(   rV   rW   rX   rY   rZ   r[   kwargsoutputslast_hidden_statesr;   r:   outputs                r*   forwardColPaliForRetrieval.forward   s    V#%+N%;%>%>TZZ%>%PF>"1B1N-TXT_T_TqTq %9$D $++JjJj 	 &1%<k$++B]B](( 
)%!%#/
 
 %2226../AB
  //b$/"GG
":":2">>
 ]WQR[0F%9%Eq	4F1I9E9Q'55W[]F2J'+'7D7V#CVC(!#333G'//T))?K?W ; ;
 	
 ^b
 	
r-   c                 6    U R                   R                  5       $ N)rN   get_input_embeddingsr(   s    r*   ro   (ColPaliForRetrieval.get_input_embeddings   s    xx,,..r-   c                 :    U R                   R                  U5        g rn   )rN   set_input_embeddings)r(   values     r*   rs   (ColPaliForRetrieval.set_input_embeddings   s    %%e,r-   c                 6    U R                   R                  5       $ rn   )rN   get_output_embeddingsrp   s    r*   rw   )ColPaliForRetrieval.get_output_embeddings   s    xx--//r-   c                 :    U R                   R                  U5        g rn   )rN   set_output_embeddings)r(   new_embeddingss     r*   rz   )ColPaliForRetrieval.set_output_embeddings   s    &&~6r-   c                 :    U R                   R                  U5        g rn   )rN   set_decoder)r(   decoders     r*   r~   ColPaliForRetrieval.set_decoder   s    W%r-   c                 6    U R                   R                  5       $ rn   )rN   get_decoderrp   s    r*   r   ColPaliForRetrieval.get_decoder       xx##%%r-   c                 6    U R                   R                  5       $ rn   )rN   tie_weightsrp   s    r*   r   ColPaliForRetrieval.tie_weights   r   r-   new_num_tokenspad_to_multiple_ofmean_resizingc                 <   U R                   R                  UUUS9nUR                  U R                  R                  R
                  l        UR                  U R                  R                  l        UR                  U R                   l        UR                  U l        U$ )N)r   r   r   )rN   resize_token_embeddingsnum_embeddingsr   r   r   rK   )r(   r   r   r   model_embedss        r*   r   +ColPaliForRetrieval.resize_token_embeddings   s     xx77)1' 8 
 9E8S8S**5,8,G,G)*99&55r-   )rM   r   rO   rQ   rN   rK   )NNNNNN)NNT)r/   r0   r1   r2   r   rJ   r   r   rA   
LongTensorrB   rD   boolr   r   r8   rk   ro   rs   rw   rz   r~   r   r   intr   r&   r   r6   __classcell__)rT   s   @r*   rF   rF   c   s   } $  154815,0/3&*3
E,,-3
 u0013
 !.	3

 $D>3
 'tn3
 d^3
 
u//	03
 3
j/-07&&&
 )-,0"	  %SM 	
 
 r-   rF   )r@   dataclassesr   typingr   r   r   r   rA   r   transformersr	   cache_utilsr   modeling_utilsr   utilsr   r   configuration_colpalir   r   r8   rF   __all__r.   r-   r*   <module>r      s     ! / /   4   - 0 0 
 ?_ ? ?, $< $< $<N 
o0 o
of r-   