a
    hdf                     @   s>  U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5 ddl6m7Z7 e8e9Z:ej;e<d< g dZ=G dd de'Z>G dd de$Z?G dd de?Z@d2eAeBe	f eeCe#dddZDeEe# eEe# dddZFeAeBe	f eCe&d d!d"ZGd3eEe& eCeHeEe& ef d#d$d%ZIeed&d'd(ZJeeeCd)d*d+ZKejLeeCd,d-d.ZMeEe& eeCd/d0d1ZNdS )4    N)ChainMap)reduce)AnycastOptionalUnion)narrow_tensor_by_index)dedup_save_plans)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors)set_element)BytesStorageMetadataChunkStorageMetadataMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESStorageMetaTensorStorageMetadata)LoadPlanLoadPlannerReadItemSavePlanSavePlanner	WriteItemWriteItemType)_compare_save_plans_contains_usable_plan"_create_default_metadata_only_plan_create_read_items_create_write_items_init_state_dict_merge_delta_local_plans)find_state_dict_object)DTensor   )_versionlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc                   @   s6  e Zd ZU eed< d%eeee eeddddZd&eee	 eddd	d
Z
edddZee ee dddZee eee ef dddZee eee ee ef dddZee eee ef dddZeedddZeedddZeeejejf dddZeedd d!Zeed"d#d$Z dS )'r)   mappingsTNF)r   flatten_sharded_tensorsdedup_replicated_tensorsdedup_save_to_lowest_rankenable_plan_cachingreturnc                 C   s>   || _ || _i | _|| _|d ur*td | jj| _|| _	d S )NzDefaultSavePlanner's `dedup_replicated_tensors` argument is being deprecated, and no longer has any effect. Please remove this argument from your call.)
r   r0   r/   r2   r(   warning	__class____name___cached_plans_key_enable_plan_caching)selfr   r0   r1   r2   r3    r;   Z/var/www/auris/lib/python3.9/site-packages/torch/distributed/checkpoint/default_planner.py__init__H   s    
zDefaultSavePlanner.__init__)
state_dictstorage_metais_coordinatorr4   c                 C   s2   | j rt |\}| _| jr"t|}|| _|| _d S N)r   r/   r0   r   r>   r@   )r:   r>   r?   r@   r;   r;   r<   set_up_planner]   s    z!DefaultSavePlanner.set_up_plannerr4   c                 C   sv   t | j| j}| jr$tj|| jd}|| _| jrp| j	t
jv rdt|t
j| j	 rdtd tg ddS |t
j| j	< | jS )Nplanner_datazINo change in the local plan. Skipping sending the plan to the coordinatorFusable)r-   r>   r@   r   dataclassesreplacer/   planr9   r8   r   Z_cached_save_planr   r(   infor   )r:   rJ   r;   r;   r<   create_local_planj   s"    
z$DefaultSavePlanner.create_local_plan	all_plansr4   c                 C   s   t || jS rA   )r	   r2   )r:   rN   r;   r;   r<   _dedup_save_plans   s    z$DefaultSavePlanner._dedup_save_plansc                 C   s^   |  |}t|\}}| jrDdd |D }tt| }tj||d}t||sVtd||fS )Nc                 S   s   g | ]
}|j qS r;   rD   ).0pr;   r;   r<   
<listcomp>       z:DefaultSavePlanner._create_global_plan.<locals>.<listcomp>rD   zFailed to validate global plan)	rO   r.   r   dictr   rH   rI   _validate_global_plan
ValueError)r:   rN   Zdeduped_plansglobal_planmetadataZplanner_data_dictZmerged_mappingsr;   r;   r<   _create_global_plan   s    

z&DefaultSavePlanner._create_global_planc                 C   s$  g }| j tjvrL|tj| j < | |\}}|tj| j < |tj| j < |||fS t|stg ddgt| }tj| j  }tj| j  }nt	tj| j  |}|tj| j < | |\}}| j | jv rt
tj| j  |D ]0\}}t||r|tg dd q|| q|tj| j < |tj| j < |||fS )zw
        Create global plan with caching.
        Returns a tuple of global_plan_delta, global_plan, metadata.
        FrF   )r8   r   Z_cached_all_plansrY   Z_cached_global_planZ_cached_metadatar   r   lenr#   zipr   append)r:   rN   global_plan_deltarW   rX   Zmerged_plansZcached_plannew_planr;   r;   r<    _create_global_plan_with_caching   s4    

z3DefaultSavePlanner._create_global_plan_with_cachingc                 C   sD   g }| j r| |\}}}n| |\}}|}|| _|| _|| jfS rA   )r9   r_   rY   rW   rX   )r:   rN   r]   rW   rX   r;   r;   r<   create_global_plan   s    z%DefaultSavePlanner.create_global_planr^   r4   c                 C   s,   |}|j stj| j }n|}|tj| j< |S rA   )rG   r   Z_cached_final_save_planr8   r:   r^   Zfinished_planr;   r;   r<   _finish_plan_with_caching   s    z,DefaultSavePlanner._finish_plan_with_cachingc                 C   s    |}| j r| |}|| _| jS rA   )r9   rc   rJ   rb   r;   r;   r<   finish_plan   s
    
zDefaultSavePlanner.finish_plan)
write_itemr4   c                 C   s   |  |j}| ||S rA   )lookup_objectindextransform_object)r:   re   objectr;   r;   r<   resolve_data   s    zDefaultSavePlanner.resolve_datarg   r4   c                 C   s   t | j|S zSExtension from the planner interface to make it easy to extend the default planner.r$   r>   r:   rg   r;   r;   r<   rf     s    z DefaultSavePlanner.lookup_object)re   ri   c                 C   s(   |j tjkr$t }t|| |}|S rl   )typer   BYTE_IOioBytesIOtorchsave)r:   re   ri   bytesr;   r;   r<   rh     s
    z#DefaultSavePlanner.transform_object)TTNFF)NF)!r7   
__module____qualname__r
   __annotations__boolr   r=   r   r   rB   r   rL   listrO   tupler   rY   r_   r`   rc   rd   r   r   rs   Tensorrq   rr   rj   r   r   rf   rh   r;   r;   r;   r<   r)   E   sL   
       <
	r)   c                   @   s   e Zd ZU dZeed< eed< d%eeedddd	Zd&ee	e
 edd
ddZedddZee ee dddZeedddZeejddddZedddZeejddddZeejdd d!Zeejd"d#d$ZdS )'r*   ak  
    DefaultLoadPlanner that adds multiple features on top of LoadPlanner.

    In particular it adds the following:

    flatten_state_dict: Handle state_dict with nested dicts
    flatten_sharded_tensors: For FSDP in 2D parallel mode
    allow_partial_load: If False, will raise a runtime error if a key is present in state_dict, but not in the checkpoint.
    original_state_dictr/   TFN)r   r0   allow_partial_loadr4   c                 C   s"   || _ || _i | _i | _|| _d S rA   )r   r0   r}   r/   r~   )r:   r   r0   r~   r;   r;   r<   r=     s
    zDefaultLoadPlanner.__init__r>   rX   r@   r4   c                 C   sF   t | || _| jrt|}| jr0t|\}| _|| _|| _|| _d S rA   )	r"   r}   r0   r   r   r/   r>   rX   r@   )r:   r>   rX   r@   r;   r;   r<   rB   (  s    z!DefaultLoadPlanner.set_up_plannerrC   c                 C   s   | j d usJ | jrzt| j }t| j j }|| }|rzdt_t| j\}}t| }||@ rt|| | _| _	d t_t
| j| j | j S )NZ2_3)rX   r   setr>   keysstate_dict_metadatar'   Z_derived_versionr}   r/   r+   r~   )r:   Zcurrent_keysZ	load_keysZmissing_keysZold_state_dictZold_mappingsZold_keysr;   r;   r<   rL   ;  s"    z$DefaultLoadPlanner.create_local_plan)rW   r4   c                 C   s   t |S rA   )r,   )r:   rW   r;   r;   r<   r`   b  s    z%DefaultLoadPlanner.create_global_planra   c                 C   s   |S rA   r;   )r:   r^   r;   r;   r<   rd   e  s    zDefaultLoadPlanner.finish_plan)	read_itemvaluer4   c                 C   sF   | j r*t| j| j|jj tj|dd ntj|dd| j|jj< d S )NF)Zweights_only)	r   r   r}   r/   
dest_indexfqnrs   loadr>   )r:   r   r   r;   r;   r<   
load_bytesh  s    zDefaultLoadPlanner.load_bytes)r   c                 C   s   |  |j}| ||S rA   )lookup_tensorr   transform_tensorr:   r   tensorr;   r;   r<   resolve_tensort  s    z!DefaultLoadPlanner.resolve_tensor)r   r   r4   c                 C   s   d S rA   r;   r   r;   r;   r<   commit_tensorx  s    z DefaultLoadPlanner.commit_tensorrk   c                 C   s   t | j|S rl   rm   rn   r;   r;   r<   r   {  s    z DefaultLoadPlanner.lookup_tensor)r   r   c                 C   s   t ||j|jS rl   )r   Zdest_offsetslengthsr   r;   r;   r<   r     s    z#DefaultLoadPlanner.transform_tensor)TTF)NF)r7   rv   rw   __doc__r   rx   r
   ry   r=   r   r   rB   r   rL   rz   r`   rd   r   rq   rr   r   r   rs   r|   r   r   r   r   r;   r;   r;   r<   r*     s8   

     'r*   c                       sR   e Zd ZdZd fdd	ZeeedddZde	e
e edd	 fd
dZ  ZS )_EmptyStateDictLoadPlannera  
    Extension of DefaultLoadPlanner, which rebuilds state_dict from the saved metadata.
    Useful for loading in state_dict without first initializing a model, such as
    when converting a DCP checkpoint into a Torch save file.

    . N.B. `state_dict` must be an empty dictionary when used with this LoadPlanner

    .. warning::
        Because the entire state dict is initialized, It's recommended to only utilize
        this LoadPlanner on a single rank or process to avoid OOM.

    Nc                    s   || _ t j|i | d S rA   )r   superr=   )r:   r   argskwargsr6   r;   r<   r=     s    z#_EmptyStateDictLoadPlanner.__init__)keyrX   r4   c                    s|    j d u rdS | j v rg }|j|}|D ]0}|rR|d|d t|g q,|| q,t fdd|D rxdS dS )NT.c                 3   s   | ]}| j v V  qd S rA   )r   )rP   unflattened_keyr:   r;   r<   	<genexpr>  rS   zA_EmptyStateDictLoadPlanner._should_include_key.<locals>.<genexpr>F)r   rE   getr\   joinstrany)r:   r   rX   Zunflattened_keysrE   r   r;   r   r<   _should_include_key  s    

z._EmptyStateDictLoadPlanner._should_include_keyFr   c                    s   |rJ |d usJ |j  D ]d\}}| ||s4qt|trRtj|j|jj	d}|j
d urz||j
v rzt||j
| | q|||< qt ||| d S )N)dtype)r   itemsr   
isinstancer   rs   emptysize
propertiesr   rE   r   r   rB   )r:   r>   rX   r@   kvr   r;   r<   rB     s    

z)_EmptyStateDictLoadPlanner.set_up_planner)N)NF)r7   rv   rw   r   r=   r   r   ry   r   r   r   rB   __classcell__r;   r;   r   r<   r     s     r   T)r>   rX   strictr4   c                 C   s   g }|   D ]\}}||jvr6|rtd| dnq|j| }t|trt|dd d ur|j| krtd|j d|  d| t|tr|j	
 d ur|t|||7 }q|t|||7 }qt|S )Nz&Missing key in checkpoint state_dict: r   r   zSize mismatch between saved z and current: z for )r   r   RuntimeErrorr   r   getattrr   rV   r%   device_meshget_coordinater    r   )r>   rX   r   requestsr   objmdr;   r;   r<   r+     s*    


r+   rM   c                 C   s   | S )z
    Create global load plan used by DefaultLoadPlanner.

    The default load behavior involved no global coordination and this function
    currently doesn't change the local plans.
    r;   )rN   r;   r;   r<   r,     s    	r,   )r>   r@   r4   c                 C   sT   g }|   D ]>\}}t|tr<|j durJ|t||7 }q|t||7 }qt|S )a  
    Create the ``SavePlan`` used by DefaultSavePlanner.

    On non-coordinator ranks, this function ignores tensors and non-tensor objects,
    only producing writes for ShardedTensor objects.

    On the coordinator rank, produce writes for all values.
    N)r   r   r%   r   r   r!   r   )r>   r@   r   r   r   r;   r;   r<   r-     s    
r-   )rN   rewrite_index_hintsr4   c           
      C   s"  i }g }| D ]}g }|j D ]}|jtjks<|jj|vs<J |jtjkrbt ||jj< || q|j	duspJ t
t||jjt|j	j|j	jg d}|}|rtj|jt|jd}	tj||	d}|| |j	jdusJ d|jj d|j|j	j q|tj||d q|t|fS )a6  
    Create the global plan and metadata used by DefaultSavePlanner.

    Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans.

    The only global planning change is to update index hints in all ``MetadataIndex`` objects if
    ``rewrite_index_hints`` is True.
    N)r   r   chunks)rg   zZ
                    Cannot create MD for tensor without bounds.
                    FQN: z
                )r   )r   ro   r   ZSHARDrg   r   rp   r   r\   Ztensor_datar   r   
setdefaultr   r   rH   rI   rZ   r   chunkr   )
rN   r   r   Z	new_plansrJ   Z	new_itemsitemZ	tensor_mdZnew_itemZ	new_indexr;   r;   r<   r.     sF    



r.   )r>   r4   c                 C   s   t | }t|g\}}|S )zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)r   r.   )r>   rJ   _r   r;   r;   r<   _create_default_local_metadataK  s    r   )box0box1r4   c                 C   sd   t | j}t|D ]L}| j| |j| |j|  kr: dS |j| | j| | j|  kr dS qdS )z9Check if two boxes overlap. Tuples are (offset, lengths).FT)rZ   offsetsrangesizes)r   r   Zndimsir;   r;   r<   _check_box_overlapR  s    
r   )outer_box_size	inner_boxr4   c                 C   s`   t t| D ]N}|j| dk r$ dS |j| dk r8 dS |j| |j|  | | kr dS qdS )Nr   FT)r   rZ   r   r   )r   r   r   r;   r;   r<   _check_box_boundsb  s    r   )rW   rX   r4   c           
   	   C   s   d}|j  D ]\}}t|tr"qt|jdkr2qd}t|jD ]t\}}t|j|sjt	
d||j| d}|ttj|jd7 }|j|d d  D ]"}t||rt	
d||| d}qq@ttj|jd}	||	krt	
d||	| d}q|S )NTr   z~
                        key:%s has out of bounds chunk:
                        tensor-size:%s chunk: %s
                    Fr&   z$key:%s has overlapping chunks: %s %szq
                    key:%s invalid fill tensor-volume:
                    %s chunks-volume: %s
                )r   r   r   r   rZ   r   	enumerater   r   r(   r5   r   operatormulr   r   )
rW   rX   all_goodr   r   Zchunks_volumeZ	chunk_idxZchunk0Zchunk1Ztensor_volumer;   r;   r<   rU   p  sB    
	
	rU   )T)T)OrH   rq   loggingr   collectionsr   	functoolsr   typingr   r   r   r   rs   Ztorch.distributed._shard._utilsr   Z.torch.distributed.checkpoint._dedup_save_plansr	   Z)torch.distributed.checkpoint._nested_dictr
   r   Z2torch.distributed.checkpoint._sharded_tensor_utilsr   Z&torch.distributed.checkpoint._traverser   Z%torch.distributed.checkpoint.metadatar   r   r   r   r   r   r   r   Z$torch.distributed.checkpoint.plannerr   r   r   r   r   r   r   Z,torch.distributed.checkpoint.planner_helpersr   r   r   r    r!   r"   r#   Z"torch.distributed.checkpoint.utilsr$   Ztorch.distributed.tensorr%    r'   	getLoggerr7   r(   Loggerrx   __all__r)   r*   r   rT   r   ry   r+   rz   r,   r-   r{   r.   r   r   Sizer   rU   r;   r;   r;   r<   <module>   s^   
(
$	$	 JvB + 5