o
    Zhl                     @   s6  d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZmZmZmZ ddlmZ e	 r5ddlmZ eeZG dd deZ	d%d
dZd&defddZ					d'ddZd(ddZ					d'ddZdd Z							d)ddZ						d*ddZ	d+ddZd,dd Z d,d!d"Z!d,d#d$Z"dS )-z#PyTorch - TF 2.0 general utilities.    N   )
ExplicitEnumcheck_torch_load_is_safeexpand_dimsis_numpy_arrayis_safetensors_availableis_torch_tensorloggingreshapesqueezetensor_size)	transpose)	safe_openc                   @   s    e Zd ZdZdZdZdZdZdS )TransposeTypez
    Possible ...
    nosimpleZconv1dZconv2dN)__name__
__module____qualname____doc__NOSIMPLECONV1DCONV2D r   r   U/var/www/auris/lib/python3.10/site-packages/transformers/modeling_tf_pytorch_utils.pyr   ,   s    r    c                 C   s  |dur%|  |sd| vrtd|  d| d| t|d } | d} | dd} td	d
| } | dd} tdd| } | d} t| dkrP| dd } t|}| d dkrh|durht|dkrht	j
}n+| d dkr||dur|t|dkr|t	j}nt| d dv pd| v pd| v rt	j}nt	j}| d dks| d dks| d dkrd| d< | d dkrd| d< | d dks| d dkr| d dd| d< d| } |r| |dd} | |fS )aU  
    Convert a TF 2.0 model variable name in a pytorch model weight name.

    Conventions for TF2.0 scopes -> PyTorch attribute names conversions:

        - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch)
        - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList)

    return tuple with:

        - pytorch model weight name
        - transpose: `TransposeType` member indicating whether and how TF2.0 and PyTorch weights matrices should be
          transposed with regards to each other
    NZfinal_logits_biaszWeight name z  does not start with name_scope z. This is an internal error in Transformers, so (unless you were doing something really evil) please open an issue to report it!/z:0r   z/[^/]*___([^/]*)/z/\1/z_._z//+r   kernel      )r   pointwise_kerneldepthwise_kernelZ	emb_projsZ	out_projsZ
embeddingsgammaweightbetabiasr"   r#   Z_kernelz.weight.)
startswith
ValueErrorlenlstripreplaceresubsplitlistr   r   r   boolr   r   join)Ztf_namestart_prefix_to_removetf_weight_shape
name_scoper   r   r   r   (convert_tf_weight_name_to_pt_weight_name7   sR   

  $
r7   Tr   c              
   C   s   | t ju r|r	dnd}t||d}n| t ju rt|dd}n	| t ju r't|}|du r-|S t|t|jk r;t|}nt|t|jkrJt|dd}t	|t	|jkrszt
||}W |S  tyr } z| j||f7  _|d}~ww |S )z
    Apply a transpose to some weight then tries to reshape the weight to the same shape as a given shape, all in a
    framework agnostic way.
    )   r!   r   r   )r!   r8   r   r   )axes)r8   r   r   Nr   )Zaxis)r   r   transpose_funcr   r   r+   shaper   r   r1   r
   AssertionErrorargs)r   r%   Zmatch_shapept_to_tfr9   er   r   r   apply_transpose   s.   



r@   Fc              	   C   s   zddl }ddl}ddlm}	 W n ty   td  w t|tr&|g}i }
|D ]*}t	j
|}td|  |drD|	|}nt  |j|ddd	}|
| q*td
tdd |
 D dd t| |
|||||dS )*Load pytorch checkpoints in a TF 2.0 modelr   N)	load_fileLoading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.zLoading PyTorch weights from z.safetensorscpuT)Zmap_locationZweights_onlyzPyTorch checkpoint contains c                 s   s    | ]}|  V  qd S N)Znumel).0tr   r   r   	<genexpr>   s    z7load_pytorch_checkpoint_in_tf2_model.<locals>.<genexpr>,z parameters	tf_inputsallow_missing_keysoutput_loading_info_prefixtf_to_pt_weight_rename)
tensorflowtorchZsafetensors.torchrB   ImportErrorloggererror
isinstancestrospathabspathinfoendswithr   loadupdatesumvalues!load_pytorch_weights_in_tf2_model)tf_modelZpytorch_checkpoint_pathrK   rL   rM   rN   rO   tfrQ   Zsafe_load_filept_state_dictrX   Zpt_path
state_dictr   r   r   $load_pytorch_checkpoint_in_tf2_model   s>   



&re   c                 C   s   |  }t| |||dS )rA   )rK   rL   )rd   r`   )ra   pt_modelrK   rL   rc   r   r   r   load_pytorch_model_in_tf2_model   s   rg   c              	      s\   z
ddl }ddl W n ty   td  w  fdd| D }t| ||||||dS )z*Load pytorch state_dict in a TF 2.0 model.r   NrC   c                    s2   i | ]\}}||j  jkr| n|  qS r   )dtypeZbfloat16numpyfloat)rF   kvrQ   r   r   
<dictcomp>   s    &z5load_pytorch_weights_in_tf2_model.<locals>.<dictcomp>rJ   )rP   rQ   rR   rS   rT   items$load_pytorch_state_dict_in_tf2_model)ra   rc   rK   rL   rM   rN   rO   rb   r   rm   r   r`      s*   

r`   c                 C   s   t |dkrtd| d| d| d| d	 n	td| d t | dkr5td	| d
|  d ntd| d| d t |dkr_ddd |D }td| d| d d S d S )Nr   zSSome weights of the PyTorch model were not used when initializing the TF 2.0 model : ,
- This IS expected if you are initializing z from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing z from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).z6All PyTorch model weights were used when initializing .
z,Some weights or buffers of the TF 2.0 model zH were not initialized from the PyTorch model and are newly initialized: o
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.All the weights of z were initialized from the PyTorch model.
If your task is similar to the task the model of the checkpoint was trained on, you can already use * for predictions without further training.
c              	   S   s*   g | ]\}}}d | d| d| dqS )z- z: found shape z in the checkpoint and z in the model instantiatedr   )rF   keyZshape1Zshape2r   r   r   
<listcomp>$  s    z%_log_key_warnings.<locals>.<listcomp>Some weights of zh were not initialized from the model checkpoint are newly initialized because the shapes did not match:
)r+   rS   warningr3   )missing_keysunexpected_keysmismatched_keys
class_nameZmismatched_warningr   r   r   _log_key_warnings  sH   
r   c	                     s  ddl }	|du rj}|du rd}|r.|	| |dd W d   n1 s)w   Y  i }
| D ]r}d}d|v rB|dd}d|v rL|dd	}d
|v rV|d
d}d|v r`|dd}|d}d |ddd ddgkry|d d  n|ddd ddgkr|d d   dur|dd  g }d|}|du r|}||
|< q4d}tfdd|
 D sjd }j	j
 }d}t|
 }g }g }t|d}|D ]}|j}t|||j|d\ }|dur| }|D ]
}||
v r|  nq|d   |
vr&|r|  qԈjdurt fddjD rqt  d|
  }|r3||}n|| }z	t|||j}W n0 |	jjyp } z!|s\t|}|d7 }|	j|| |j|jf W Y d}~qd}~ww |t|7 }||	||j ~|  qtd|dd  t|}jdurjD ]fd!d"|D }qj durj D ]fd#d"|D }q|st!|||j"j#d$ |r|||d%}|fS S )&zLoad a pytorch state_dict in a TF 2.0 model. pt_state_dict can be either an actual dict or a lazy-loading
    safetensors archive created with the safe_open() function.r   Nr   FZtrainingr$   r%   r&   r'   Zrunning_varZmoving_varianceZrunning_meanZmoving_meanr(   r8   parametrizations	original0_g	original1_vc                 3       | ]	}|  jV  qd S rE   r)   base_model_prefixrF   s)ra   r   r   rH   i      z7load_pytorch_state_dict_in_tf2_model.<locals>.<genexpr>
get_tensor)r4   r5   r6   c                 3   s     | ]}t | d uV  qd S rE   r.   search)rF   pat)namer   r   rH     s    z not found in PyTorch modelz_
	You may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method.zLoaded rI   z  parameters in the TF 2.0 model.c                        g | ]}t  |d u r|qS rE   r   rF   rk   r   r   r   ry          z8load_pytorch_state_dict_in_tf2_model.<locals>.<listcomp>c                    r   rE   r   r   r   r   r   ry     r   r   r|   r}   r~   )$rP   dummy_inputsr6   keysr-   r0   r3   anyr   Ztrainable_weightsZnon_trainable_weightssethasattrr   r7   r;   append_keys_to_ignore_on_load_missingAttributeErrorr   r@   errorsZInvalidArgumentErrorrV   r   Zassigncastrh   discardrS   rZ   r1   "_keys_to_ignore_on_load_unexpectedr   	__class__r   ) ra   rc   rK   rL   rM   rN   rO   ignore_mismatched_sizesskip_logger_warningsrb   Ztf_keys_to_pt_keysrx   Znew_keykey_componentsr4   Zsymbolic_weightsZtf_loaded_numelZall_pytorch_weightsr|   r~   Zis_safetensor_archiveZsymbolic_weightZsw_namer   aliasesaliasZstate_dict_namearrayr?   	error_msgr}   loading_infor   )r   r   ra   r   rp   1  s   









rp   c                 C   s   g }|D ],}	t |	dd}
t| |
||d|||dd	\} }W d    n1 s&w   Y  || qttjdd |D  }tdd |D g }tdd |D g }t|||| jj	d	 |rg|||d
}| |fS | S )Nrb   )Z	frameworkT)rK   rL   rM   rN   rO   r   r   c                 S   s   g | ]}t |d  qS )r|   )r   rF   rZ   r   r   r   ry         zAload_sharded_pytorch_safetensors_in_tf2_model.<locals>.<listcomp>c                 S      g | ]}|d  qS )r}   r   r   r   r   r   ry         c                 S   r   )r~   r   r   r   r   r   ry     r   r   r   )
r   rp   r   sortedr   intersectionr^   r   r   r   )ra   Zsafetensors_shardsrK   rL   rM   rN   rO   r   Zall_loading_infosZshardZsafetensors_archiver   r|   r}   r~   r   r   r   -load_sharded_pytorch_safetensors_in_tf2_model  s6   
r   c                 C   s   z
ddl }ddl}W n ty   td  w ddl}ddlm} td|  d| j	j
 }	t||	}
|
| j}|du rA|j}|durK||dd	 ||| t| |||d
S )z
    Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see
    https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357).
    r   NLoading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.r   )load_tf_weightsz Loading TensorFlow weights from ZTFFr   rL   rM   )rP   rQ   rR   rS   rT   transformersZmodeling_tf_utilsr   rZ   r   r   getattrconfigr   load_tf2_model_in_pytorch_model)rf   Ztf_checkpoint_pathrK   rL   rM   rb   rQ   r   r   Ztf_model_class_nameZtf_model_classra   r   r   r   $load_tf2_checkpoint_in_pytorch_model  s.   


r   c                 C   s   |j }t| |||dS )z$Load TF 2.0 model in a pytorch modelr   )weights!load_tf2_weights_in_pytorch_model)rf   ra   rL   rM   r   r   r   r   r     s   r   c                 C   sN   z
ddl }ddl}W n ty   td  w dd |D }t| |||dS )z.Load TF2.0 symbolic weights in a PyTorch modelr   Nr   c                 S   s   i | ]}|j | qS r   )r   ri   )rF   	tf_weightr   r   r   rn   .  r   z5load_tf2_weights_in_pytorch_model.<locals>.<dictcomp>r   )rP   rQ   rR   rS   rT   $load_tf2_state_dict_in_pytorch_model)rf   Z
tf_weightsrL   rM   rb   rQ   tf_state_dictr   r   r   r   "  s   r   c                    s"  dd l }i }t }d}tfdd| D s jd }i }| D ]\}	}
t|	||
jd\}}|
|f||< q&t	| }i }g }| D ]\}}|
 |v rb|
 dkrb||
  ||< qI|}|d}d }	|dd d d	d
gkr}|d d }	n|dd d d	dgkr|d d }	|	d ur|d d |	g }d|}||vr|r|| qIt| d|| \}}t|||jdd}t|rt|}t|st|s| }t|r||}|||< |||
 < || qIj|dd\}}||7 }jd urjD ]  fdd|D }qjd ur*jD ]  fdd|D }qt|dkrJtdjj d| djj djj d	 ntdjj d t|dkrktdjj d| d ntd jj d!jj d" td#|  |r||d$}|fS S )%Nr   r   c                 3   r   rE   r   r   )rf   r   r   rH   =  r   z7load_tf2_state_dict_in_pytorch_model.<locals>.<genexpr>r(   )r4   r5   r   r8   r   r   r   r   r   r   z not found in TF 2.0 modelF)r>   )strictc                    r   rE   r   r   r   r   r   ry   |  r   z8load_tf2_state_dict_in_pytorch_model.<locals>.<listcomp>c                    r   rE   r   r   r   r   r   ry     r   zSSome weights of the TF 2.0 model were not used when initializing the PyTorch model rq   rr   z from a TF 2.0 model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a TFBertForPreTraining model).
- This IS NOT expected if you are initializing z from a TF 2.0 model that you expect to be exactly identical (e.g. initializing a BertForSequenceClassification model from a TFBertForSequenceClassification model).z5All TF 2.0 model weights were used when initializing rs   rz   zG were not initialized from the TF 2.0 model and are newly initialized: rt   ru   z were initialized from the TF 2.0 model.
If your task is similar to the task the model of the checkpoint was trained on, you can already use rv   z1Weights or buffers not loaded from TF 2.0 model: )r|   r}   ) rQ   dictZnamed_parametersr   r   r   ro   r7   r;   r   Zdata_ptrr0   r3   r   r   r@   ri   Zisscalarr   r   r   Z
from_numpyr   Zload_state_dictr   r   r+   rS   r{   r   r   rZ   )rf   r   rL   rM   rQ   Znew_pt_params_dictZcurrent_pt_params_dictr4   Ztf_weights_mapr   r   Zpt_namer   Zall_tf_weightsZloaded_pt_weights_data_ptrZmissing_keys_ptZpt_weight_nameZ	pt_weightZpt_weight_name_to_checkr   r   r|   r}   r   r   )r   rf   r   r   4  s   











r   )r   NN)NT)NFFNN)NF)NFFNNFF)NFFNNF)NFF)FF)#r   rW   r.   ri   utilsr   r   r   r   r   r   r	   r
   r   r   r   r:   Zsafetensorsr   Z
get_loggerr   rS   r   r7   r@   re   rg   r`   r   rp   r   r   r   r   r   r   r   r   r   <module>   s`   0

H-

3
#,
 
3

)
	