o
    Zh;                  	   @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlm  m  mZ d dlmZmZmZ e eZG dd dZd	e
jjd
eeeeee f f fddZ dS )    N)defaultdict)Iterator)contextmanager)Enum)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                   @   s   e Zd ZU G dd deeZeeZe	eef e
d< e Zee e
d< edddZeed	eded fd
dZededdfddZdS )SimpleProfilerc                   @   s$   e Zd ZdZdZdZdZdZdZdS )zSimpleProfiler.TypeallZ
all_gatherZall_gather_objectZ
reshardingH2DD2HN)	__name__
__module____qualname__ALLZ	ALLGATHERZALLGATHER_OBJZ
RESHARDINGr   r    r   r   R/var/www/auris/lib/python3.10/site-packages/torch/distributed/fsdp/_debug_utils.pyType   s    r   results	profilingreturnNc                 C   s   | j   | j  d S )N)r   clearr   )clsr   r   r   reset"   s   
zSimpleProfiler.resetprofile_typec                 c   s    || j vsJ | d| j | t }zd V  W t }| j|  || 7  < | j | d S t }| j|  || 7  < | j | w )Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r   addtime	monotonicr   remove)r   r   beginendr   r   r   profile'   s   zSimpleProfiler.profilemsgc                 C   s8   t  dkrt  t jjkrtd|| j |   d S )Nr   z%s %s)	distZget_rankZget_debug_levelZ
DebugLevelINFOloggerinfor   r   )r   r"   r   r   r   dump_and_reset9   s   zSimpleProfiler.dump_and_reset)r   N)r   r   r   strr   r   r   floatr   dict__annotations__setr   classmethodr   r   r   r!   r'   r   r   r   r   r	      s   
 r	   modelr   c                 C   s:   dd }dd }dg}i }t | ||dd |  D ||S )a  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line reprents a submodule name that contats the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                    s  |d }t  dkr d dkr d d n }|d | jj d }d| | }t| }	|	d u r<|d  |d 7  < d S |	j| d }
|
rR|d  |d	 d 7  < n
|d  |d 7  < |
r|
j}t|tj	siJ  fd
d|j
D }||v r|| | d S |||< d S d S )N   r   .[] 
z FULLY SHARDEDc                    s   g | ]}t  | qS r   )r   ).0nameprefixr   r   
<listcomp>   s    zX_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fn.<locals>.<listcomp>)len	__class__r   r   Z_fully_sharded_module_to_handlegetZ
flat_param
isinstanceflat_param_fileZFlatParameterZ_fqnsextend)moduler9   Z
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacesZtrimed_prefixZprefixed_module_nameZprinted_prefixed_module_namestatehandleparamZglobal_fqnsr   r8   r   	module_fnj   s2   &

zD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnc                 S   s   | d |fS )Nr   r   )rB   rC   r   r   r   	return_fn   s   zD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn c                 S   s   g | ]\}}|qS r   r   )r6   key_r   r   r   r:      s    zE_get_sharded_module_tree_with_module_name_to_fqns.<locals>.<listcomp>)r   Znamed_parameters)r.   rH   rI   rB   rC   r   r   r   1_get_sharded_module_tree_with_module_name_to_fqnsB   s   ($rM   )!loggingr   collectionsr   collections.abcr   
contextlibr   enumr   ZtorchZtorch.distributeddistributedr#   Z"torch.distributed.fsdp._flat_paramZfsdpZ_flat_paramr?   Z$torch.distributed.fsdp._common_utilsr   r   r   	getLoggerr   r%   r	   nnModuletupler(   r*   listrM   r   r   r   r   <module>   s"   
,