o Zh@sddlZddlZddlZddlZddlmZmZmZddlm Z m Z m Z m Z m Z mZddlZddlmZddlmmmZddlmmmZddlmmmZddlmZddlm Z ddl!m"Z"m#Z#ddl$m%Z%ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2dd l3m4Z4dd l5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mm?Z?dd l@mAZAdd lBmCZCddlDmEZEe rddlFmGZGdZHz ddlImJZJmKZKWn eLydZHYnweMdZNdZOePejQejQfZRe eejQeRfZSe;jTe2jTe;jUe2jUe;jVe2jVe;jWe2jWe;jXe2jXiZYe;jWe;jXgZZe;jVe;jXfZ[e dde(deSde;de e?de e#de(f ddZ\e de(deSde#de(fddZ]e de de^fdd Z_e de#de^fd!d"Z`e d#eMdejQfd$d%Zae d&ejQd#eMdejQfd'd(Zbd&ejQd#eMdePejQejQffd)d*Zce dde(d+ejdd,e eejjdd-ee eejjee eejjdfde(f d.d/Zfd-ege d0e^ddfd1d2Zhe de(d+ejdd3eiejed4e eeMejjfde(f d5d6Zke de(d+ejdde(fd7d8Zle de(de e;d9e e:d:e e7d;e^deMde(fd?d@Zme de(de(fdAdBZne de(dCe6dDe^de(fdEdFZoe dde(de#de(fdGdHZpe de(de(fdIdJZqd+ejddKegejeddfdLdMZre de(dNejdd4e eeMejjfdOe e ejdgdfdPe^de(f dQdRZse de(dKegejedNejdfdSdTZtdUejddVe eejjddeiejdfdWdXZu ddUejjdd,eiejjddYe eejjedeiejjefdZd[ZvdUejjdd,eiejjddeiewfd\d]ZxdUejddeiewfd^d_Zyd+ejdd3eiejed4e eeMejjfddfd`daZzd4e eeMejjfdbeMdce'de ejjfdddeZ{d+ejdd3eiejed,eiejddePe^e^ffdfdgZ|dUejddOe ejdgdfd,eiejdddfdhdiZ}dUejddje ejjd,eiejddce'fdkdlZ~dUejdd,eiejddegejdfdmdnZd+ejdd3eiejedoeiejdje ejjddf dpdqZdKegejedregejdje ejjddfdsdtZdudvZd+ejdd3eiejedje ejjdbeMdce'dejjf dwdxZd+ejddKegejedejQddfdydzZd{egejddfd|d}Zd+ejdd3eiejedeejefd~dZd3eiejeddfddZde;fddZdejQde jfddZdS)N) GeneratorIterableIterator)AnyCallable no_type_checkOptional TYPE_CHECKINGUnion) default_hooks)_mesh_resources DeviceMesh)_get_default_group)_FSDPDeviceHandle _FSDPState_get_module_fsdp_state_is_fsdp_flattened!_named_parameters_with_duplicatesclean_tensor_name TrainingState)_FSDP_USE_FULL_PREC_IN_EVAL FlatParameterFlatParamHandleHandleShardingStrategy)_FreeEventQueue)BackwardPrefetch CPUOffloadFullOptimStateDictConfigFullStateDictConfigMixedPrecisionShardingStrategyStateDictConfig StateDictType)_Policy)DTensorExtensions)_sync_params_and_buffers)is_traceable_wrapper_subclass)RemovableHandleT) deferred_initfakeFiZ _fsdp_syncedstate process_groupsharding_strategypolicy device_meshreturncCs|dur |dur td|tv}|r-|dur&|dur&|dur&td|dt|||}n|r:||_|jdd|_n |dur@|nt|_|j|_|j|_ |j }|r\||j 9}t j ||_||j|_|S)NzcCannot pass both process_group and device_mesh at the same time. Please just pass only one of them.zManual wrapping with zA requires explicit specification of process group or device_mesh.rZmesh_dim) ValueErrorHYBRID_SHARDING_STRATEGIES*_init_process_group_state_for_hybrid_shard _device_mesh get_groupr+rranksize world_size_inter_node_pgr DefaultStateZ_get_gradient_predivide_factorZ_gradient_predivide_factorZ_gradient_postdivide_factor)r*r+r,r-r.Zis_hybrid_strategyZdata_parallel_world_sizer;Q/var/www/auris/lib/python3.10/site-packages/torch/distributed/fsdp/_init_utils.py_init_process_group_stateYs<   r=cCs|r t|r||_|jdd|_|jdd|_n4td|j|dur8t}t||j \}}||_||_nt |rC|\|_|_n tdt |t |jd|_|S)Nrr0z,Expected device_mesh to have ndim=2 but got zmExpected process_group to be passed in as either None or Tuple[dist.ProcessGroup, dist.ProcessGroup] but got r+)"_is_valid_hybrid_shard_device_meshr4r5r9r+r1ndimr!_init_intra_and_inter_node_groups_device_handleZ device_count_is_valid_hybrid_shard_pg_typetype_get_default_comm_hook_stateZ_inter_node_state)r*r+r.Z default_groupZintra_node_groupZinter_node_groupr;r;r<r3s4  r3cCs(t|tot|dkotdd|DS)Ncs|] }t|tjVqdSN) isinstancedist ProcessGroup).0Zpgr;r;r< z1_is_valid_hybrid_shard_pg_type..)rJtuplelenallr?r;r;r<rDs  rDcCst|to |jdkS)NrG)rJr rA)r.r;r;r<r@sr@num_devices_per_nodecCst|\}}|S)aU Return a process group across the current node. For example, given each row is a distinct node: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 This API would return an intra-node subgroup across [0, 1, ..., 7] or [8, 9, ..., 15] depending on the process's rank. For example, rank 3 would get [0, 1, ..., 7]. )rKZ new_subgroups)rSZintra_node_subgroup_r;r;r<_init_intra_node_process_groups rUglobal_process_groupc sd}t|}t|}|}t|}tD]fddt|D}tj||d}|kr6|}q|dusBJ|d|S)a Return an inter-node process group where each contained rank has the same local rank. For example, given each row is a distinct node: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 This API would return inter-node process group [0, 8], [1, 9], [2, 10], and so forth depending on the process's rank. For example, rank 1 would get [1, 9], rank 5 would get [5, 13]. Ncsg|]}|qSr;r;)rMiZ local_rankrSr;r<  z2_init_inter_node_process_group..)Zranksbackendz. expected to assign inter-node pg, but did not)rK get_backendZget_world_sizeZget_rankrangeZ new_group) rVrSZ inter_node_pgZsharding_backendr8Z num_nodesZ my_local_rankZranks_for_inter_groupgrpr;rXr<_init_inter_node_process_groups"     r_cCst|t||fS)a Initialize intra and inter-node process groups and return the ones corresponding to this process's rank. This function can be used to initialize process groups for ``HYBRID_SHARD`` or ``_HYBRID_SHARD_ZERO2`` in FSDP. This function assumes each node has an equal number of CUDA-enabled devices. Returns: Tuple[dist.ProcessGroup, dist.ProcessGroup]: Intra and inter-node process group. )rUr_)rVrSr;r;r<rBsrBmoduleignored_modulesignored_statescCs|dur |dur tdd}|du}|rt|}t|dng}t|dur)t|ngdt|dkr@t|dtjr>|}n|}t|||_t ||j||_ t ||j|_ |S)NzfCannot pass both ignored_modules and ignored_states at the same time. Please just pass ignored_states.TFr) r1list_check_ignored_statesrQrJnn Parameter_get_ignored_modules_ignored_modules_get_ignored_params_ignored_params_get_ignored_buffer_names_ignored_buffer_names)r*r`rarbignored_parameterspassed_as_ignored_statesZignored_states_listr;r;r<_init_ignored_module_statess8     rorncCst|dkrdS|r6tdd|D}tdd|D}|s2|s4tdd|Dtd}td |dSdStd d|DsQtd d|Dtd}td |dS) z Check that the ignored states are uniformly parameters or uniformly modules. We may remove this check in the future if we permit mixing. rNcsrHrI)rJrerfrMr*r;r;r<rNIrOz(_check_ignored_states..csrHrIrJreModulerpr;r;r<rNJrOcSh|]}t|qSr;rErpr;r;r< Mz(_check_ignored_states..)keyzUignored_states expects all nn.Parameter or all nn.Module list elements but got types csrHrIrqrpr;r;r<rNSrOcSrsr;rtrpr;r;r<ruTrvz>ignored_modules expects nn.Module list elements but got types )rQrRsortedreprr1)rbrn all_paramsZ all_modulesZ sorted_typesr;r;r<rd>s* rdignored_params device_idcCsd}|durt|tjr|nt|}|durRt||D]%}|jjdvr%q|dur-|j}q|jj|jkrAtd|jd|jjq|pHtj}|jdkrRtdt ||_ |S)a< Determine device handle used for initializing FSDP. If a device is specified by ``device_id``, then returns device handle corresponds to that device type. Otherwise, If the module is already on a non-CPU device, then the device type is that non-CPU device type. If the module is on CPU or meta, then the device type is the current accelerator device. See the :ref:`Accelerators` for details. This method will be called once ignored paramters was determined, as the device handle maybe needed for other initialization. N>metacpuzLFSDP does not support modules with different device types but got params on z and r~zOFSDP needs a non-CPU accelerator device, but no accelerator device is detected.) rJtorchdevice_get_orig_paramsrE RuntimeError_CZ_get_acceleratorrZ from_devicerC)r*r`r{r|Zdetermined_deviceparamr;r;r<_init_device_handle[s8    rcCs<t||_i}|D] \}}t|}|j||<q ||_|SrI)_get_buffer_namesZ _buffer_names named_buffersrZdtype_buffer_name_to_orig_dtype)r*r`r buffer_namebufferr;r;r<_init_buffer_states  rmixed_precision cpu_offloadlimit_all_gathersuse_orig_paramsbackward_prefetch_limitforward_prefetch_limitc Cs |jdkr|tjkrtd|ptjdtj}n |tjkr'tjdtdd|p+tj|_|p1t|_ |durCt j dt |j tjtdd k|_|pQt|_||_||_tj|_d|_t|_t|_t !|j|||_"d|_#i}||_$d} | |_%g} | |_&|S) Nr>z/FSDP is switching to use `NO_SHARD` instead of z since the world size is 1.zoThe `NO_SHARD` sharding strategy is deprecated. If having issues, please use `DistributedDataParallel` instead.) stacklevelz'torch.distributed.fsdp.mixed_precision.1)'r8r NO_SHARDwarningswarn FULL_SHARD FutureWarningr,rrrrZ_log_api_usage_oncestrosenvirongetrZ_use_full_prec_in_evalrrr_use_orig_paramsrZIDLEZtraining_stateZ_is_rootrZ_free_event_queuerKZget_debug_levelZ _debug_levelexec_order_utilsZ_ExecOrderDataZ_exec_order_dataZ_unshard_event_fully_sharded_module_to_handle_handleparams) r*r,rrrrrrrrrr;r;r<_init_core_statesT       rcCs4g}||_g}||_g}||_d|_d|_d|_|S)NT)_root_pre_forward_handles_pre_forward_handles_post_forward_handlesZ_sync_gradientsZ _comm_hookZ_comm_hook_state)r*rrrr;r;r<_init_runtime_statesrbackward_prefetchforward_prefetchcCs||_||_|SrI)rr)r*rrr;r;r<_init_prefetching_statesrcCs2t|}|r||jkrt|j|_|Sd|_|SrI)r Z get_root_meshr4r$rC_fsdp_extension)r*r.Z root_meshr;r;r<_init_extensions  rcCs*tj|_t}t|_||_i}||_|SrI)r"ZFULL_STATE_DICTZ_state_dict_typerrZ_optim_state_dict_configZ_state_dict_configZ_unshard_params_ctx)r*Zstate_dict_configZunshard_params_ctxr;r;r<_init_state_dict_statesrrcCsZ|D](}t|jdkr*d}|D] \}}||ur|}nq|s"Jtd|dqdS)z Verify if the parameters are accepted by FSDP. The only restriction now is that the parameter cannot be a scalar tensor (param.shape == []). rrz/FSDP doesn't support scalar parameters. Change z& to a 1D tensor with numel equal to 1.N)rQshapeZnamed_parametersr1)r`rr param_namenameZparam_r;r;r<_verify_managed_paramss rfully_sharded_module param_init_fnsync_module_statesc st|j|t|jj}t|jj\}}|s|r)|dur)t||jn|r5t||jjn |rBt j |fdddddjD}t |j||t |j|jj_ tt|j} t|| |rt|| jjtvrt|| jt| |S)zHInitialize a ``FlatParamHandle`` from a module ``fully_sharded_module``.Ncst|duo |jvSrI)rrh) submoduler*r;r<Ls z0_init_param_handle_from_module..)Zcheck_fncSh|] }|D]}|qqSr;buffers)rMZignored_modulerr;r;r<ruPsz1_init_param_handle_from_module..)_check_single_device_modulerj_get_device_from_device_idr6rC_need_to_materialize_modulerh_materialize_with_param_init_fn_materialize_meta_moduler(Zmaterialize_module_move_module_to_device_get_compute_devicecompute_devicercrr_sync_module_params_and_buffersr+r,r2r9_init_param_handle_from_params) r*rr|rrdevice_from_device_idis_meta_moduleis_torchdistX_deferred_initignored_buffersmanaged_paramsr;rr<_init_param_handle_from_module-sf       rc Cst|dkrdSt|||jt|j|jj|jj|jj |jj |j |j |j d }||jr/J|j|j||_||j|j<td}|jjrU|jj|krW||dSdSdS)Nr)Zfsdp_extensionr~)rQrrSHARDING_STRATEGY_MAPr,rZoffload_paramsrZ param_dtypeZ reduce_dtypeZkeep_low_precision_gradsr+rrZshardrrappendZ flat_paramrZ_fully_sharded_modulerrZ flat_param_to)r*rrhandle cpu_devicer;r;r<rrs0    r root_modulerhc Csd}z |dur t|nt}Wnty'}z t|dt||d}~ww|D]}t|tjjs>t|dt|t|rFtdq*| D] }t |sW| |qKdd|D}||vrkt d|| D]}t|}|durt|d sJ||jqo|S) ah Check that ``_ignored_modules`` is an iterable of ``nn.Module`` s without any FSDP instances. Return the modules contained in their module subtrees as a :class:`set`. Nested FSDP instances are excluded, but their already-computed ignored modules are included. ``_ignored_modules`` represents the argument passed by the user to FSDP. z>`ignored_modules` should be an iterable of `torch.nn.Module`s Nzbut got zbut got an iterable with z1`ignored_modules` should not include FSDP modulescSs*h|]}|D] }t|tjs|qqSr;)modulesrJ fsdp_fileFullyShardedDataParallel)rMr`childr;r;r<rus z'_get_ignored_modules..zTrying to ignore the top-level module passed into the FSDP constructor itself will result in all parameters being ignored and is not well-supported: rh)set TypeErrorrErJrrerrrr1rtraversal_utilsZ _composableaddrrhasattrupdaterh) rrhZ msg_prefixZignored_root_moduleser`raroptional_fsdp_stater;r;r<rgsD      rgrmcCszt}dd|D}|||durdd|D}|||D]}t|}|dur:t|ds4J||jq#|S)z Return the parameters of the modules in ``ignored_modules`` and the parameters in ``ignored_parameters``. :class:`FlatParameter` s are excluded from the result. cSs&h|]}|D]}t|s|qqSr;) parametersr)rMmpr;r;r<rusz&_get_ignored_params..NcSsh|]}t|s|qSr;)rrMrr;r;r<rus rj)rrrrrrj)rrarmZall_ignored_paramsZparams_in_ignored_modulesZparams_in_ignored_parametersrrr;r;r<ris"     ricslt}dd|D|fdd|D|D]}t|}|dur3t|ds-J||jq|S)z6Return the cleaned buffer FQNs in ``ignored_modules``.cSrr;r)rMrrr;r;r<rus z,_get_ignored_buffer_names..cs h|] \}}|vrt|qSr;r)rMrrZbuffers_in_ignored_modulesr;r<rus Nrl)rrrrrrrl)rraZall_ignored_buffer_namesrrr;rr<rks    rkcCsdd|DS)zrReturn the fully prefixed names of all buffers in the module hierarchy rooted at ``root_module`` as a class:`set`.cSsh|]\}}t|qSr;r)rMrrTr;r;r<ru rZz$_get_buffer_names..)r)rr;r;r<r srcCs`ddt||D}t|dkr!td|vr!|durtddSt|dkr.td|dS) z Raise an error if ``module`` has original parameters on multiple devices, ignoring the parameters in ``ignored_params``. Thus, after this method, the module must be either fully on the CPU or fully on a non-CPU device. cSsh|]}|jqSr;rrMrr;r;r<rusz._check_single_device_module..rGr~NzTTo support a module with both CPU and GPU params, please pass in device_id argument.r>z;FSDP only supports single device modules but got params on )rrQrrr)r`r{r|Zdevicesr;r;r<rs  rr6 device_handlec Csv|durdSt|tjr|nt|}|jdkr9|jdur9td|d|d|d|jd t|}|S)z Return a ``torch.device`` for the specified ``device_id``. Processes ``device_id`` and returns either the corresponding device or ``None`` if ``device_id`` is ``None``. Nr~z"FSDP got the argument `device_id` z on rank zJ, which does not have an explicit index. FSDP will use the current device z6. If this is incorrect, please explicitly call `torch.zk.set_device()` before FSDP initialization or pass in the explicit device index as the `device_id` argument.)rJrrrEindexrrcurrent_device)r|r6rrr;r;r<r.s  rcCsxtt||}tdd|D}|D]}||vrq|jddD]}||jO}q!q| o7to7tdd|D}||fS)z Return if ``module`` has parameters on meta device and if ``module`` is using torchdistX deferred initialization. At most of the returned bools can be ``True``. If either is ``True``, then ``module`` needs to be materialized. css|]}|jVqdSrI)is_metarr;r;r<rNXsz._need_to_materialize_module..Frecursecss|]}t|VqdSrI)r)Zis_fakerr;r;r<rNds)rcranyrrr_TORCHDISTX_AVAIL)r`r{rarrrbufrr;r;r<rKs   rcCs@t|std|dt|t||}|D]}||qdS)Nz Expected z to be callable but got )callabler1rE_get_modules_to_materialize)rrramodules_to_materializer`r;r;r<ris  rrc Cs|pt|}t||}d}zAt2|D]%}t|jdd|jdd}t t |dk}|r=|j |dd| qWdWdS1sJwYWdSt yo} ztdt| dt|d| d} ~ ww)NFrr)rrzIUnable to call `reset_parameters()` for module on meta device with error z(. Please ensure that your module oftype z* implements a `reset_parameters()` method.)rrrrno_grad itertoolschainrrrQrcZto_emptyZreset_parameters BaseExceptionrrrrE) rrrarZmaterialization_devicerr`Zmodule_state_iterZhas_module_statesrr;r;r<rws:  & rcCsrg}t|g}|h}|r7|}|||D]}||vr4t|dur4||vr4||||q|s |SrI) collectionsdequepopleftrchildrenrr)rrarqueueZvisited_modulesr`Z child_moduler;r;r<rs       rrc std|durjt}||g}g}|rP|}|fdd|jddD|fdd|jddD| D] }t |t j sM||q@|sfdd |D} fd d |D} t | | |dStt|d} | dur| jkrtdSdSdS) a Move ``module`` depending on ``device_from_device_id`` and its current device. This includes moving ignored modules' parameters. - If ``device_from_device_id`` is not ``None``, then this moves ``module`` to the device. - If ``device_from_device_id`` is ``None``, then this does not move ``module`` but warns the user if it is on CPU. Precondition: ``_check_single_device_module()``. r~Nc3|] }|jkr|VqdSrIrrrr;r<rN z)_move_module_to_device..Frc3rrIr)rMrrr;r<rNrcg|]}|vr|qSr;r;r)r{r;r<rYz*_move_module_to_device..crr;r;r)rr;r<rYr)rrrrrrextendrrrrJrr_move_states_to_devicenextr_warn_cpu_init) r`r{rrrrrZ curr_modulerZparams_to_moveZ bufs_to_moverr;)rrr{r<rs6         rrc Cst|dkrt|dkrdSt|dkr|dj}n t|dkr%|dj}td}|durg|D])}t|||_|jdurJ|j||j_Wdn1sTwYq0|D]}|||_q\dS||krptdSdS)z Move states to the specified device. Precondition: ``_check_single_device_module()`` and module's parameters and buffers have been materialized if needed. rNr~)rQrrrtodataZgradr)rrrrrrrr;r;r<rs,          rcCstddS)NamThe passed-in `module` is on CPU and will thus have FSDP's sharding initialization run on CPU, which may be slower than on GPU. We recommend passing in the `device_id` argument for FSDP to move `module` to GPU for the sharding initialization. `module` must also be on GPU device to work with the `sync_module_states=True` flag since that requires GPU communication.)rrr;r;r;r<rsrcCshtt||d}|dur|jjdkr|j}nt|}|dur2||kr2td|d|d||S)a) Determine and return this FSDP instance's compute device. If the module is already on a non-CPU device, then the compute device is that non-CPU device. If the module is on CPU, then the compute device is the current device. Since this method should be called after materializing the module, any non-CPU device should not be meta device. For now, the compute device is always a CUDA or CUDA-like device with its explicit index. Precondition: ``_check_single_device_module()`` and ``_move_module_to_device()``. Nr~z4Inconsistent compute device and `device_id` on rank z: z vs )rrrrErrr1)r`r{rr6rrrr;r;r<rsrc sg}|D]0}t|tds6t|td|tr1\}}fdd|D}||q|q|D]$}|trX\}}fdd|D} || q9|q9t |t ||t dddS) z Synchronize module states (i.e. parameters ``params`` and all not-yet-synced buffers) by broadcasting from rank 0 to all ranks. Precondition: ``sync_module_states == True`` and ``self.process_group`` has been set. FTcg|]}t|qSr;getattrrMattr)detached_bufferr;r<rYIz3_sync_module_params_and_buffers..crr;rr)detached_paramr;r<rYRr r)srcN) rr FSDP_SYNCEDsetattrdetachr&Z__tensor_flatten__rr+_check_module_states_for_sync_module_statesr%PARAM_BROADCAST_BUCKET_SIZE) r`rr+ module_statesrattrsrTZ inner_buffersrZ inner_paramsr;)r r r<r4s2          rrcCs&|rtdd|DrtddSdS)Ncss |] }|jtdkVqdS)r~N)rr)rMZtensorr;r;r<rNcs z>_check_module_states_for_sync_module_states..zThe module has CPU parameters or buffers when `sync_module_states=True`, which requires them to be on GPU. Please specify the `device_id` argument or move the module to GPU before passing it to FSDP.)rr1)rr;r;r<r`s rccsB|}z t|}||vrt|s|Vqty YdSw)aD Return an iterator over the original parameters in ``module``. The iterator does not return the parameters in ``ignored_params``, any ``FlatParameter`` s (which may be present due to nested FSDP wrapping), or any original parameters already flattened (only relevant when ``use_orig_params=True``). TN)rrr StopIteration)r`r{Z param_genrr;r;r<rms  rcCsFt|D]\}}||vr t|s td|d|d|jqdS)a5 Check that original parameters in ``fsdp_module`` have been flattened. The flattened parameters are made invisible to ``named_parameters()`` for the module hierarchy rooted at ``fsdp_module``. This should be called as a sanity check after flattening the wrapped module's parameters. z Found an unflattened parameter: z;  N)rrrr7 __class__)Z fsdp_moduler{rrr;r;r<_check_orig_params_flatteneds rcCs|tjkrtjStjSrI)r rr Zallreduce_hookZreduce_scatter_hook)r,r;r;r<_get_default_comm_hooks rcCs tj|dS)Nr?)r r:r?r;r;r<rFs rFrI)rrrrcollections.abcrrrtypingrrrrr r rZtorch.distributed distributedrKZ(torch.distributed.fsdp._exec_order_utilsZfsdpZ_exec_order_utilsrZ'torch.distributed.fsdp._traversal_utilsZ_traversal_utilsrZ2torch.distributed.fsdp.fully_sharded_data_parallelZfully_sharded_data_parallelrZtorch.nnreZ(torch.distributed.algorithms._comm_hooksr Ztorch.distributed.device_meshr r Z"torch.distributed.distributed_c10drZ$torch.distributed.fsdp._common_utilsrrrrrrrZ"torch.distributed.fsdp._flat_paramrrrrZ%torch.distributed.fsdp._limiter_utilsrZtorch.distributed.fsdp.apirrrrrr r!r"Ztorch.distributed.fsdp.wrapr#Z&torch.distributed.tensor.parallel.fsdpr$Ztorch.distributed.utilsr%Ztorch.utils._python_dispatchr&Ztorch.utils.hooksr'rZ torchdistxr(r) ImportErrorintrrrPrLZHybridShardProcessGroupTypeZProcessGroupTyperrZ SHARD_GRAD_OPZ HYBRID_SHARDZ_HYBRID_SHARD_ZERO2rr2Z#NO_RESHARD_AFTER_FORWARD_STRATEGIESr=r3boolrDr@rUr_rBrrrfrorcrdrrrrrrrrrrrrrgrirrkrrrrrrrZTensorrrrrrrrrrr:rFr;r;r;r<s     $ (       3 + $  .  0  A     D  <   "        #  6    # ,