o Zh@sZUddlZddlZddlZddlZddlmZmZddlmZm Z m Z ddl m Z ddl mZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZmZmZm Z m!Z!m"Z"ddl#m$Z$dd l%m&Z&m'Z'm(Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1dd l2m3Z3dd l4m5Z5dd l6m7Z8ddl9m:Z:gdZ;dZdZ?e@eAZBee3eejCeDeEeAfZFeeFeGeFeHeFeIeAdffZJeIeAeJfZKeGeKZLeIeAeeKeLffZMe@ZNe@eeOd<ejPddZQe GdddZRe GdddeRZSejT   dcdejUdeAd eAd!eVd"eVd#eBf d$d%ZWGd&d'd'ZXddd(d)ZYddd*dejUd+eHejZj[d,fd-eVd.ee@ejUd/eeRd#eSf d0d1Z\d2eIeAeJfd3eMd4eSd#dfd5d6Z]d7eejUejZj[fd8eAd#efd9d:Z^d;eIeAefd4eSd#eIeAeffdd?Zae`dejUd;eIeAeJfd4eSd#e5fd@dAZbdBejZj[d#dfdCdDZcd;eMd#eIeAeJffdEdFZddBejZj[d;eIeAeJfd4eSd#eMfdGdHZee`dejUdIeHejZj[d,fd4eSd#eMfdJdKZfdejUdBejZj[d3eMd4eSd#eMf dLdMZge`dejUdIeHejZj[d,fd;eMd4eSd#df dNdOZhddd*dejUd.ee@ejUd/eeRd#eIeAeJffdPdQZiddd*dejUdIeejZj[eejZj[fd.ee@ejUd/eeRd#eMf dRdSZjddd*dejUdIeejZj[eejZj[fd.ee@ejUd/eeRd#eHeIeAeJfeMff dTdUZkdejUd;eeIejUeIeAeJffeIeAeJffd#eIeAeJffdVdWZlddXdejUd2eIeAeJfd/eeRd#e5fdYdZZmddXdejUdIeejZj[eejZj[fd3eMd/eeRd#df d[d\ZnddXdejUdIeejZj[eejZj[fd2eIeAeJfd3eMd/eeRd#e5f d]d^ZoeddXdejUd/eeRd#dfd_d`ZpeddXdejUdIeHejZj[d,fd/eeRd#dfdadbZqdS)eN) GeneratorIterable)asdict dataclassfield)chain)AnyCallablecast no_type_checkOptionalUnion) ShardedTensor)_broadcast_state_dict_distribute_state_dict_flatten_state_dict_gather_state_dict_offload_state_dict_to_cpu_unflatten_state_dict)_CHECKPOINT_PREFIX)FullOptimStateDictConfigFullStateDictConfigFullyShardedDataParallelOptimStateDictConfigShardedOptimStateDictConfigShardedStateDictConfigStateDictConfig StateDictType)._get_module_fsdp_state_if_fully_sharded_moduleFSDP_WRAPPED_MODULE)DTensor)_IncompatibleKeys)DistributedDataParallel) tree_map_only) FQNS_T PrimitiveType ValueType DictValueTypeListDictValueTypeOptimizerStateTypeStateDictOptionsget_model_state_dictget_optimizer_state_dictget_state_dictset_model_state_dictset_optimizer_state_dictset_state_dictZ _flat_param param_groupsparamsstater&_patched_state_dictccs@t}tzdVW|rtdSdS|rtwwN)gc isenableddisableenable) is_enabledr;V/var/www/auris/lib/python3.10/site-packages/torch/distributed/checkpoint/state_dict.py _gc_contextQs  r=c@sreZdZUdZdZeed<dZeed<dZeed<dZ eed<dZ eed<dZ eed <dZ eed <d Z eed <d S)r*ap This dataclass specifies how get_state_dict/set_state_dict will work. - ``full_state_dict``: if this is set to True, all the tensors in the returned state_dict will be gathered. No ShardedTensor and DTensor will be in the returned state_dict. - ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if ``full_state_dict`` is also true, then only the rank0 will get the state_dict and all other ranks will get empty state_dict. - ``ignore_frozen_params``: if the value is True, the returned state_dict won't contain any frozen parameters -- the ``requires_grad`` is False. The default value is False. - ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option indicates whether to keep the submodule prefixes from the state_dict keys. or example, if the submodule is ``module.pretrain`` and the full FQN of the parameter is ``pretrain.layer1.weight`` of the param. When this option is True, the parameter's key in the returned state_dict will be ``pretrain.layer1.weight``. If the options is False, the key will be ``layer1.weight``. Note that if ``keep_submodule_prefixes`` is False, there may be conflicted FQNs, hence there should be only one submodule in ``submodules``. - ``strict``: the ``strict`` option when ``set_state_dict`` calls model.load_state_dict(). - ``broadcast_from_rank0``: when the option is True, rank0 should receive a full state_dict and will broadcast the tensors in the state_dict/ optim_state_dict one by one to other ranks. Other ranks will receive the tensors and shard according to the local shards in the model and optimizer. ``full_state_dict`` must be set to True when using this option. This option currently only supports DTensor, not the legacy ShardedTensor. Ffull_state_dict cpu_offloadignore_frozen_paramsTkeep_submodule_prefixesstrictbroadcast_from_rank0flatten_optimizer_state_dict_fqn_modifiersdsd_fqn_modifiersN)__name__ __module__ __qualname____doc__r>bool__annotations__r?r@rArBrCrDrFstrr;r;r;r<r*\s  $      r*c@seZdZUeedZeeeej fee ej ffe d<eedZ eeeej fee ej ffe d<ee dZe ee d<dZee d<dZee d<ejZee d<eedZeeje d <d S) _StateDictInfo)default_factoryfqn_param_mappingshared_params_mappingsubmodule_prefixesT handle_model handle_optim fsdp_context fsdp_modulesN)rGrHrIrdictrPr rMtorchTensorr$rLrQsetrRrSrKrT contextlib nullcontextrUr listrVnnModuler;r;r;r<rNs$       rNrETmodelnamerFskip_ddp_prefixskip_compiler_prefixreturnc s|td}d|vr |hS|d}g}|}t|D]\}} t|tr4| dks)J|j}|s3|| qt|trz|t |dkrf||dt krfd |t |t } rZdfdd| j DSt |t}| tkry|| t || }qt|tjjjr| dksJ|j}|s|| qt||rt ||| } rt|| rt || }|| | tjjjkr|t |dkrtdqt || }qd |tdhS) a This API is used to convert the name of a parameter to the FQNs. For FSDP without `use_orig_params`, the name of FlatParameter can be mapped to multiple original parameters. As a result, the return type of this function is `set[str]`. Args: module (nn.Module): the root model. name (str): the name skip_ddp_prefix (bool): whether to skip DDP's `module` prefix Returns: The canonical FQNs based on the model traversal. .modulecsh|]}|qSr;r;.0fqnprefixr;r< z_get_fqns.. _orig_modz-Expect `_extra_state` to be the last obj name)replacersplit enumerate isinstanceDDPrgappendFSDPlen _FLAT_PARAMjoingetattrZ_fqnsrrXZ_dynamoZ eval_frameZOptimizedModulerphasattrgetr^modules_EXTRA_STATE_KEY_SUFFIX RuntimeError) r`rarFrbrcZ obj_namesZ fqn_obj_namesZcurr_objiZ curr_obj_nameZ flat_paramZ removed_fqnr;rlr< _get_fqnssX                     rc@s eZdZdS) _EXTRA_STATEN)rGrHrIr;r;r;r<rsrc#s<tdtjdtdtffdd |dEdHdS)Nrgcurr_fqnrdc3s||r |dnd}|D]-\}}|vrqt|r2|t|vr2|dd}n||}||EdHqt|jdd|jddD]\}}||jvrXqN||}||fVqNt|j dt j j t j j kr|t j jj}|tfVdSdS)NrfreF)recurseget_extra_state)addZnamed_childrenr|r{valuesrZ named_buffersnamed_parametersZ_non_persistent_buffers_set __class__r^r_rr~rgrr)rgrra submodulenew_fqnobjrFrZvisited_modulesr;r<rs2     z+_iterate_valid_model_state..recursere)rZr^r_rMr)r`rFr;rr<_iterate_valid_model_states "r) submodulesoptionsoptims. optim_onlyrrc Cs&|rtdt|r|std|pt}i}i}t|D]@\}}t|tr'qt||} | |d} | durIt t t || | ||||<n| ||<| D] } t|ts\||| <qQqt|D]\} } | D] } t tj| || <qjqdt } |rt |}|D]"\}}||vrqt||} t| dksJd| dd| Dq|jr|jstdt|}|r|jrt|j|jd }t|j|jp|jd }tj}nt|jd }t |jd }tj!}t"j#d d }t$j%|||||d }nt"j&}t'dit(|||| |t tt)j*|| t|dkdS)zW Verify the model and options passed by the user and generates _StateDictInfo. zGetting submodules only model/optim state_dict is deprecated and will be removed in 2.5. This feature can be achieved by manually filtering out the state_dict returned from get_state_dict.z;Optimizers are not passed in but optim_only is set to True.Nrhz)Submodule FQN should only have 1 instancecss|]}|dVqdS)rfNr;rir;r;r< Lsz"_verify_options..z?full_state_dict must be True when broadcast_from_rank0 is True.)offload_to_cpuZ rank0_only)rc sst5tjddtdtj||||d dVWdn1s%wYWddSWddS1s=wYdS)NignorezFSDP.state_dict_type)messagecategoryrgstate_dict_typestate_dict_configoptim_state_dict_config)warningscatch_warningsfilterwarnings FutureWarningrwrrr;r;r<$fsdp_state_dict_type_without_warningjs  "z=_verify_options..fsdp_state_dict_type_without_warningrr)rPrQrRrUrVrSrTr;)+rwarnrrr*rrtrrr}r rZrMupdatecopyr]itemsrXrY named_modulesrxrCr> ValueErrorrwrVrr?rrZFULL_STATE_DICTrrZSHARDED_STATE_DICTr[contextmanager functoolspartialr\rNrr^r_)r`rrrrrPrQraparamfqnsrkZparam_Zfqns_rRrgrVrrrrrUr;r;r<_verify_optionss              rmodel_state_dictoptim_state_dictinfocCs|jD]}t|}|dusJdq|jr3|s3|js3|js3|jr#|js3|jr3|js3t dt d|j rH|sH|jr>|jsH|jsHt d|| D]}t|vr\t |dtdqLdS)Nz)Expected a fsdp_state with a fsdp module.z}The option indicates that model state_dict is required to save or load, but model state_dict is empty.rank = dist.get_rank()=rfzgThe option indicates that model state_dict is required to save, or load but optim state_dict is empty. z contains z6. This can happen if the model is not the root module.)rVrrSrRr@r?r>rBrCrdistget_rankrTkeysry)rrrrgZ fsdp_statekeyr;r;r<_verify_state_dictsZ   rrapicCs,t||}|tvrtjt|j||d}|S)N)self)r{r4rrr)rrcallr;r;r<_state_dict_fns r state_dictcCs@|jr|jr tjs dnd}t||j|dS|jrt|S|S)Nr;)r)r? ranks_only)r>r?rX distributedZis_initializedrr)rrrr;r;r<_maybe_full_or_cpu_state_dictsrc Cs|jsiS|t|d}Wdn1swYt|D]:}t||}t|dks8J||ftt|}||kr_dt fdd}|||sXt d|d|| |||<q%|j ri}|D]&}|j D] }| |svqn|jr||||<qn|t|d} |||| <qnqi|}|jr|D]\}} | jrqt||}|D]}| |qqt|D]\}} t| r| jr| |qt||S)NrrhrdcSst|t|kr dS|d}|d}d}t|D]&\}}|||kr9|d7}|t|kr8|t|dkSq|dvr>qdSdS)NFrfrrh)rgrpT)rxrrrs)rrkZ fqn_splitZ key_splitZfqn_idxZkey_idxZkey_namer;r;r<verifys    z%_get_model_state_dict..verifyzAn unexpected key, z, exists. FQN is )rSrUrr]rrrxnextiterrKrpoprR startswithrAr@r requires_gradrrX is_tensoris_metar) r`rrrrrkrnew_state_dictrmrrpr;r;r<_get_model_state_dictsP           rcCs|jr|s |js tiiSi}t||jD]J\}}t|||j}t|||jddd}t||D]/\}} |jrrZrrXrdimrrremoverxZdistributed_c10dZ_get_pg_default_devicerrr?rrUr r)r`rrlocal_state_dictrvaluerZfqns_with_prefixrkZfqn_with_prefixZ load_valuerZdevicesZ local_stater;r;r<_load_model_state_dictsv          $roptimcCs|jrdS|jD]}|tD] }|jdurdSqq|jD]}|tD] }|jr/t||_q$qg}|jD]}d|vrT||dt|dtj rPt dnd|d<q6|j dd|jD] }d|vrk| d|d<q^|j dddS)zH Initialize optim states by calling the step() with zero grads. Nlrg)closurerT)Z set_to_none)r3r1_PARAMSZgradrrXZ zeros_likervrtrYZtensorsteprZ zero_grad)r param_grouprZlrsr;r;r<_init_optim_statecs:          rc Csdd}i}tt|tD] \}}tt|D]\}}||||td|d|<qqtt|tD]&}|t}ttt |D]}|D]\}}||td|d|<qKqEq7|S)aI This API flattens the optimizer state_dict to support optimizer resharding for MPMD, e.g., pipeline parallelism. Without the API, the original optimizer state_dict looks like: { "state": { "layer1.weight": { "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor }, "layer2.weight": { "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor }, }, "param_group": [ { "lr": 0.0, "betas": (0.9, 0.95), ..., "params": ["layer1.weight", "layer2.weight"] } ] } With this API, the optimizer state_dict looks like: { "state.layer1.weight.step": 10, "state.layer2.weight.step": 10, "state.layer1.weight.exp_avg": SomeTensor, "state.layer2.weight.exp_avg": SomeTensor, "state.layer1.weight.exp_avg_sq": SomeTensor, "state.layer2.weight.exp_avg_sq": SomeTensor, "param_group.layer1.weight.lr" : 0.1, "param_group.layer2.weight.lr" : 0.1, "param_group.layer1.weight.betas" : (0.9, 0.95), "param_group.layer2.weight.betas" : (0.9, 0.95), } Note that if any of the value is a container, like the betas in the example, this API won't flattent it. cSs*t|tjttfstdt|ddS)NzUFlattening optimizer state_dict only supports tensor, int, float states now. Type is rf)rtrXrYintfloatNotImplementedErrortype)vr;r;r<_raise_if_type_not_supportedsz?_flatten_optim_state_dict.._raise_if_type_not_supportedrf) r r'_STATErr(_PGrrr]rM) rrretrkr3krrrr;r;r<_flatten_optim_state_dicts* rc Csi}g}t|t|i}|jD]}|tgi|tD]j}|j|D]b}||jvrGd} |D]} | tkr5q.td|d| } | |vrEd} nd} | sLq!|dt} t| t sYJ| ||j sbq!i||<|j |D]} |td|d| t t ||| <qmq!qt t t|dtd}|D]=} | tkrq|td|d| }| |dvr||d| <q|d| |krtd|d| d|d|d| d qq |S) z This API unflattens the state_dict generated by _flatten_optim_state_dict(). See the docstring of _flatten_optim_state_dict() for more detail. FrfTrrzaAll the parameters in the same parameter group should have the same saved param_group value. But z is z while other(s) is )rrr1rvrrPrQrrtr]rr3r r'rMr)rrrr3pg_state return_osdrrrk in_paramsrZ flatten_keyr2Z state_nameZfirst_param_fqnrr;r;r<_unflatten_optim_state_dictsl            r optimizersc s|jsiStitgi}|D]}t|t|d}|jrm|t|||}Wdn1s2wY|s:q t |t D]}d|vrW|t ||t| dd<qB|tD]}dd|t D}||t <q\nut tdd|jD}tt|tt|} i|D](\} } t|| } t| d ksJtt| } | | vrq| | }|| <| |<qt |t D]} | } |t | |t| <q|tD]}fd d|t D|t <q|sq tt|t|ttt|t|tq |jr ttt |}t!||S) Nrrp _orig_mod.recSsg|]}|ddqS)rrerqrjrr;r;r< 'roz)_get_optim_state_dict..css|]}|tVqdSr5)r)rjgr;r;r<r*sz(_get_optim_state_dict..rhcsg|]}|qSr;r;)rjpidZfqn_pid_mappingr;r<r<s)"rTrrrrrVrUrwrr]rrrqrr from_iterabler1rWrrangerxrrrrr r'rr(extendrDr)rr)r`rrrrZosdrrr2Zparam_pid_mappingrrrrkrgroupr;rr<_get_optim_state_dict s\          rc Cs:i}g}t|t|i}i}tddtt|tDr|S|jD]}|tgi|tD]q} |j | D]i} | |j vrXd} tt |tD]} | tt t | tvrVd} nqEnd} | s]q5|dt} t| t sjJ| | | jr}tt|t| || <tt |tD]} | tt t | tvrt|td|t| <qq5q.t|tdkrg}tt |tD]} ttt t | tdkr|| qt|dkrtdt|tt|jkrtd t|td|t| <q!tt |tD]'}|t|d}|dkrq|D]\}}|tkrq||||<qq|S) a Extract the corresponding optim state_dict from ``optim_state_dict`` for ``optim`` and return the result optim state_dict. Args: model (nn.Module): the root model. optim (torch.optim.Optimizer): the optimizer. optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that contains the optim state_dict of ``optim``. info (_StateDictInfo): state dict information. Returns: The optim state_dict of ``optim``. css|]}t|tVqdSr5)rtrrr;r;r<rfs  z*_split_optim_state_dict..FTrrhrzThere are param groups that have zero parameters. In such a case, DSD only support exactly one param group with zero parameters.But the loaded state_dict has zero or more than one param groups that have zero parameters.z`When there is a parameter group that has zero parameters, multiple optimizers are not supported.)rrallr r'rr1rvrrPrQr(r]rMrtrrxidrr}r)r`rrrr3rrZ pg_mappingrrrkrZloaded_param_groupr2rZpg_idxrrr;r;r<_split_optim_state_dictLs          rc s~|jsdS|D]4}t||r*t|vrt||||}nt|ttttf||}ni}|j r| D]d\}}t ||}t ||dd} || krHq3t |dksPJ| | |tD]} ttttf| } fdd| tD} | | t<q\tt|t} t| D]}|vr| || |<qq3|t|||}Wdn1swYn|jr4d|_t||f|}d|_dfdd}ttj||}dusJt|\}}t|\}}|jrt||d nt ||d |D]}||vr||vs J||||<||||<qt!||}|tD]}t|vr2gttttf|t<q t"|d |d qdS) NF)rcrhcsg|]}|qSr;r)rjr)rkfqn_with_compilerr;r<rs z*_load_optim_state_dict..Tcs4|dkrdur|j|S|jkrtd|S)NrzDevice mismatch)rrr)trr;r<_devices  z'_load_optim_state_dict.._devicerrr)#rTrrrrr rWrMr&rVrrrxrrrrr'r]rrqrUrwZoptim_state_dict_to_loadr>rr#rXrYrrCrrrr)r`rrrrrZ original_fqn_rZfqns_with_compilerrvalr2Z osd_staterrrZ flatten_osdZ osd_mappingZflatten_local_osdZlocal_osd_mappingZ optim_keyZpgr;)rrkrr<_load_optim_state_dicts                rcCsVtt|dd||d}t||}t|i||WdS1s$wYdS)aH Return the model state_dict of ``model``. See ``get_state_dict`` for the detail usage. Args: model (nn.Module): the nn.Module to the model. submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters that belong to the submodules. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be returned. See `StateDictOptions` for the details. Returns: The state_dict for ``model``. :rtype: typing.Dict[str, ValueType] r;FrrrN)r=rrr)r`rrrrr;r;r<r+ s  $r+cCstt-t|tjjr|fnt|}t||d||d}t|||}ti|||WdS1s3wYdS)a Return the combined state_dict for optimizers. See ``get_state_dict`` for the detail usage. Args: model (nn.Module): the nn.Module to the model. optimizers (Union[None, Optimizer, Iterable[Optimizer]]): The optimizers that are used to optimize ``model``. submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters that belong to the submodules. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be returned. See `StateDictOptions` for the details. Returns: The state_dict for ``optimizers``. :rtype: OptimizerStateType TrN) r=rtrXr Optimizertuplerrr)r`rrrrrr;r;r<r,0s    $r,cCst4t|tjjr|fnt|}t||d||d}t||}t|||}t |||||fWdS1s:wYdS)a Return the model state_dict and optimizers state_dict. ``get_state_dict`` can process any module that is parallelized by PyTorch FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any combination of these parallelisms. The main functions of ``get_state_dict`` are: 1.) returning a model and optimizer state_dict that can be resharded with a different number of trainers and/or different parallelisms. 2.) hiding the parallelism-specific state_dict APIs. Users don't have to call these APIs. 3.) sanity checking the result state_dict. The keys of the result state dictionary are the canonical FQNs (Fully Qualified Names). A canonical FQN refers to the FQN based on a parameter's position in an nn.Module hierarchy. More specifically, a canonical FQN to a parameter is the FQN returned by ``module.named_parameters()`` or ``module.named_buffers()`` when the module is not distributed by any parallelisms. Since the optimizer internally uses parameter IDs to represent a parameter, there will be a conversion from the parameter IDs to the canonical FQNs when calling this API. ``get_state_dict`` can also process a module that is not parallelized. In such a case, ``get_state_dict`` only performs one function -- converting the optimizer parameter IDs to the canonical FQNs. Example: >>> # xdoctest: +SKIP >>> import torch >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP >>> from torch.nn.parallel import DistributedDataParallel as DDP >>> from torch.distributed.checkpoint.state_dict import get_state_dict >>> fsdp_model = FSDP(copy.deepcopy(model)) >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3) >>> ddp_model = DDP(copy.deepcopy(model)) >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3) >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim) >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict( ... fsdp_model, fsdp_optim ... ) >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(), >>> # the asserts will fail. >>> assert ddp_state_dict == fsdp_state_dict >>> assert ddp_optim_state == fsdp_optim_state_dict Args: model (nn.Module): the nn.Module to the model. optimizers (Union[None, Optimizer, Iterable[Optimizer]]): The optimizers that are used to optimize ``model``. submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters that belong to the submodules. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be returned. See `StateDictOptions` for the details. Returns: ``Tuple`` that contain model state_dict and optimizer state_dict. :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType] FrN) r=rtrXrrrrrrr)r`rrrrrrr;r;r<r-]s"H    $r-c s|siSttt|tjretdtt t tjt t t ff|}i}| D]8\}}|D]/\}}||kr;q2t||}t|dksJJdtt|d|fdd| Dq2q*|St t t t f|S)NzPassing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``is deprecated and will be removed in 2.5. If you need this feature, please preprocessing the model_state_dict to achieve the same functionality.rhz/FQNs for a submodule should only have 1 elementrfcsi|] \}}||qSr;r;)rjZsubfqnrrlr;r< sz/_unflatten_model_state_dict..)rtrrrr^r_rrrr rWrMr&rrrrxr) r`rZcast_state_dictrrZsub_state_dictramrr;rlr<_unflatten_model_state_dicts,  r)rcCs\t||}tt|dd|d}t|i|t|||WdS1s'wYdS)a=Load the model state_dict. The counterpart of ``get_model_state_dict`` to set the state_dict to the model. See ``set_state_dict`` for the detail usage. Args: model (nn.Module): the nn.Module to the model. model_state_dict: (Dict[str, ValueType]): the model state_dict to load. If the key of the ``model_state_dict`` is nn.Module, the key is a submodule of ``model`` and the value should be the state_dict of the submodule. When loading the state_dict, the prefix of the submodule will be append to the state_dict. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be loaded. See `StateDictOptions` for the details. Returns: ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields: * **missing_keys** is a list of str containing the missing keys * **unexpected_keys** is a list of str containing the unexpected keys :type model_state_dict: typing.Dict[str, ValueType] r;FrrN)rr=rrr)r`rrrr;r;r<r.s  $r.cCsrt,t|tjjr|fnt|}t||d|d}ti||t||||WddS1s2wYdS)aLoad the optimizers state_dict. The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the optimizers. See ``set_state_dict`` for the detail usage. WARN: ``set_optimizer_state_dict`` can only be called before ``backward()`` or after ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be initialized correctly. Args: model (nn.Module): the nn.Module to the model. optimizers (Union[Optimizer, Iterable[Optimizer]]): The optimizers that are used to optimize ``model``. optim_state_dict: OptimizerStateType: the optimizer state_dict to load. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be loaded. See `StateDictOptions` for the details. Returns: None :type optim_state_dict: typing.OptimizerStateType TrN) r=rtrXrrrrrr)r`rrrrr;r;r<r/s  "r/cCst||}t2t|tjjr|fnt|}t||| |d}t|||t ||||t |||WdS1s=wYdS)aLoad the model state_dict and optimizers state_dict. The counterpart of ``get_state_dict`` to set the state_dict to the model and optimizers. The given ``model_state_dict`` and ``optim_state_dict`` do not have to be returned by ``get_state_dict`` but must meet the following requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``, 2) if a tensor is sharded, it must be either a ShardedTensor or DTensor, 3) optimizer state_dict cannot contain the parameter IDs; the keys should be the canonical FQNs. WARN: ``set_state_dict`` can only be called before ``backward()`` or after ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be initialized correctly. Args: model (nn.Module): the nn.Module to the model. optimizers (Union[Optimizer, Iterable[Optimizer]]): The optimizers that are used to optimize ``model``. model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]): the model state_dict to load. If the key of the ``model_state_dict`` is nn.Module, the key is a submodule of ``model`` and the value should be the state_dict of the submodule. When loading the state_dict, the prefix of the submodule will be append to the state_dict. optim_state_dict: OptimizerStateType: the optimizer state_dict to load. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be loaded. See `StateDictOptions` for the details. Returns: ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields: * **missing_keys** is a list of str containing the missing keys of the model state_dict. * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict. :type model_state_dict: typing.Dict[str, ValueType] :type optim_state_dict: typing.OptimizerStateType rN) rr=rtrXrrrrrrr)r`rrrrrr;r;r<r0+s.    $r0csjtjt||dfdd}||_tjt||ddtttfffdd }||_t |t |dS)aPatch the ``state_dict`` and ``load_state_dict`` attributes of ``model``. Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to be a partial function to call ``get_state_dict`` and ``set_state_dict``. Example: from torch.distributed.fsdp import FullyShardedDataParallel as FSDP from torch.distributed.checkpoint.state_dict import patch_model_state_dict model = fsdp(model) patch_model_state_dict(model) Args: model (nn.Module): the nn.Module to the model. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be loaded. See `StateDictOptions` for the details. Returns: None )r`rcSr5r;r;_state_dict_callr;r<state_dict_callz0_patch_model_state_dict..state_dict_callrc|ddS)N)rr;r_load_state_dict_callr;r<load_state_dict_callz5_patch_model_state_dict..load_state_dict_callN) rrr+rr.rWrMrrr4r)r`rr rr;rr r<_patch_model_state_dictms   rcstjt|||dfdd}tjt|||ddtttfffdd }t|t|t |t j j r9|fnt |}|D]}||_||_q?dS)aPatch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``. Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to be a partial function to call ``get_state_dict`` and ``set_state_dict``. Note that if there are multiple optimizers, all of the optimizers will be patched. So users only need to call one of the state_dict() to get the full result. Example: from torch.distributed.fsdp import FullyShardedDataParallel as FSDP from torch.distributed.checkpoint.state_dict import patch_model_state_dict model = fsdp(model) patch_model_state_dict(model) Args: model (nn.Module): the nn.Module to the model. options (StateDictOptions): the options to control how model state_dict and optimizer state_dict should be loaded. See `StateDictOptions` for the details. Returns: None )r`rrcr r5r;r;r r;r<r r z4_patch_optimizer_state_dict..state_dict_callrcr)N)rr;rrr;r<rrz9_patch_optimizer_state_dict..load_state_dict_callN)rrr,r/rWrMrr4rrtrXrrrrr)r`rrr rrr;rr<_patch_optimizer_state_dicts0    r)rETT)rE)rr[rr6rcollections.abcrr dataclassesrrr itertoolsrtypingrr r r r r rXZtorch.distributedrrZtorch.nnr^Z'torch.distributed._shard.sharded_tensorrZ#torch.distributed._state_dict_utilsrrrrrrZ;torch.distributed.algorithms._checkpoint.checkpoint_wrapperrZtorch.distributed.fsdprrrrwrrrrrZ$torch.distributed.fsdp._common_utilsrrZtorch.distributed.tensorr Ztorch.nn.modules.moduler!Ztorch.nn.parallelr"ruZtorch.utils._pytreer#__all__ryrrrrZrMr$rYrrr%r]rrWr&r'r(r)r4rLrr=r*rNcacher_rKrrrrrrrrrZno_gradrrrrrrrrr+r,r-rr.r/r0rrr;r;r;r<s        (       / G -    $-     C  E*@  ? ? ^ c   )  1  [$  %  , 1  B 6