o
    Zh\                     @   s  U d dl mZmZ d dlmZmZ d dlZd dlmZ	 d dl
mZ d dlmZ d dlmZmZmZ d dlmZmZ G dd	 d	eZdaee ed
< deddfddZ	d$dejdee deejee f fddZ	d$dejdedee dejfddZ	d$dejdededede	jdee dejfddZ	d$dejdededee dejf
ddZ 	d$dejdee deeje!e f fdd Z"	d$ded!ee dee dejfd"d#Z#dS )%    )ABCabstractmethod)AnyOptionalN)ShardedTensor)Shard)_all_gather_dtensor_create_chunk_dtensor_create_chunk_sharded_tensor)
DeviceMeshDTensorc                   @   s   e Zd ZdZedejdeejee	 f fddZ
edejde	dejfddZe		ddejd
edededejdeej dejfddZedejd
ededejfddZedejdeejee f fddZededee dejfddZd	S )FSDPExtensionsz
    This enables some customizable hooks to enable composability with tensor
    parallelism. To activate these hooks, use :func:`_set_fsdp_extensions` to
    set a custom :class:`FSDPExtensions` that implements the hooks.
    tensorreturnc                 C      dS )z6E.g. converting ``DistributedTensor`` to local tensor.N selfr   r   r   V/var/www/auris/lib/python3.10/site-packages/torch/distributed/fsdp/_fsdp_extensions.pypre_flatten_transform   s   z$FSDPExtensions.pre_flatten_transformparam_extensionc                 C   r   )z6E.g. converting local tensor to ``DistributedTensor``.Nr   )r   r   r   r   r   r   post_unflatten_transform   s   z'FSDPExtensions.post_unflatten_transformNrank
world_sizenum_devices_per_nodepgdevicec                 C   r   )z6Shards a tensor to chunks and returns the local chunk.Nr   )r   r   r   r   r   r   r   r   r   r   chunk_tensor(      zFSDPExtensions.chunk_tensordevice_meshc                 C   r   )zAShards a tensor/DTensor to DTensor and returns the local DTensor.Nr   )r   r   r   r   r   r   r   chunk_dtensor5   s   zFSDPExtensions.chunk_dtensorc                 C   r   )z
        This is to be called before loading a *sharded* model state dict and
        should return the tensor and list of shards from which to load data.
        Nr   r   r   r   r   pre_load_state_dict_transform?   s   	z,FSDPExtensions.pre_load_state_dict_transformparent_meshc                 C   r   )z
        This is to be called before loading a *sharded* DTensor state dict.
        This gathers tensor in FSDP dimension and returns local tensor of
        TP DTensor.
        Nr   )r   r   r"   r   r   r   all_gather_dtensorJ   r   z!FSDPExtensions.all_gather_dtensorN)__name__
__module____qualname____doc__r   torchTensortupler   r   r   r   intdistProcessGroupr   r   r   r    listr   r!   r   r#   r   r   r   r   r      sr    	
r   _extensions	flattenerr   c                 C   s   | a d S r$   )r0   )r1   r   r   r   _set_fsdp_extensions[   s   r2   r   fsdp_extensionc                 C   s.   |d ur| | \}}|d ur||fS | d fS r$   )r   )r   r3   Z
new_tensorr   r   r   r   _ext_pre_flatten_transform`   s
   r4   r   c                 C   s    |d ur|d ur| | |S | S r$   )r   )r   r   r3   r   r   r   _ext_post_unflatten_transformk   s   r5   r   r   r   r   c                 C   s"   |d ur|j nt}|| ||||S r$   )r   r
   )r   r   r   r   r   r3   Zchunk_tensor_fnr   r   r   _ext_chunk_tensoru   s   
r6   r   c                 C   s   |d ur|j nt}|| ||S r$   )r    r	   )r   r   r   r3   Zchunk_dtensor_fnr   r   r   _ext_chunk_dtensor   s   r7   c                 C   s2   |d ur	| | S t| tu sJ |  }| |fS r$   )r!   typer   Zlocal_shards)r   r3   Zshardsr   r   r   "_ext_pre_load_state_dict_transform   s
   
r9   r"   c                 C   s   |d ur|j nt}|| |S r$   )r#   r   )r   r"   r3   Zall_gather_dtensor_fnr   r   r   _ext_all_gather_dtensor   s
   
r:   r$   )$abcr   r   typingr   r   r)   Ztorch.distributeddistributedr-   Z+torch.distributed._shard.sharded_tensor.apir   Z-torch.distributed._shard.sharded_tensor.shardr   Z#torch.distributed.fsdp._shard_utilsr   r	   r
   Ztorch.distributed.tensorr   r   r   r0   __annotations__r2   r*   r+   r4   r5   r,   r.   r6   r7   r/   r9   r:   r   r   r   r   <module>   s    H




