o Zh~5@sddlZddlZddlmZddlmZmZddlZddlmZddl m Z ddl m Z ddl mZmZmZmZmZd gZejhZgd ZGd d d ejZdS) N) defaultdict)AnyOptional)nn) parametrize)type_before_parametrizations) FakeSparsityget_arg_info_from_tensor_fqnmodule_contains_param module_to_fqn swap_moduleBaseSparsifier)module module_fqn tensor_namec seZdZdZd/deeeefffdd ZdeeeffddZ d eeeeeffddfd d Z d d Z deeeffddZ d0deeefde fddZefdejdeeejddfddZddZddZ  d1deeedfdeeeeedfffd d!Zdd"efd#ejd$eeeejeejfd%e d&eejfd'd(Zd0d)e ddfd*d+Zejd#ejd,efd-d.ZZ S)2ra'Base class for all sparsifiers. Abstract methods that need to be implemented: - update_mask: Function to compute a new mask for all keys in the `groups`. Args: - model [nn.Module]: model to configure. The model itself is not saved but used for the state_dict saving / loading. - config [list]: configuration elements should be a dict map that includes `tensor_fqn` of tensors to sparsify - defaults [dict]: default configurations will be attached to the configuration. Only the keys that don't exist in the `config` will be updated. Example:: >>> # xdoctest: +SKIP("Can't instantiate abstract class BaseSparsifier with abstract method update_mask") >>> config = [{'tensor_fqn': 'layer1.weight', 'tensor_fqn': 'linear2.weight2', 'sparsity_level': 0.5}] >>> defaults = {'sparsity_level': 0.7} >>> # model.layer1.weight will have `sparsity_level` = 0.7 (getting default) >>> sparsifier = BaseSparsifier(config, defaults) Ndefaultscs.t|pi|_tt|_g|_d|_dS)NT)super__init__rrdictstategroupsenable_mask_update)selfr __class__Z/var/www/auris/lib/python3.10/site-packages/torch/ao/pruning/sparsifier/base_sparsifier.pyr7s    zBaseSparsifier.__init__returncCs|j|j|jdS)Nrrrr)rrrr __getstate__?szBaseSparsifier.__getstate__rcCs|j|dSN)__dict__update)rrrrr __setstate__FszBaseSparsifier.__setstate__cCs|jjd}t|jD]7\}}|d}|d7}|d|d7}|d|d7}t|D]}|dkr4q-|d|d||d7}q-q |d7}|S) Nz (r z Group z module: z z: ))r__name__ enumeratersortedkeys)r format_stringiZ sparse_argsrkeyrrr__repr__Is zBaseSparsifier.__repr__cCsdd|jD}|j|dS)aReturns the state of the optimizer as a :class:`dict`. It contains: * state - current state of the sparsification. * groups - a list containing all sparsity configuration groups with the key 'tensor_fqn' specifying the path to the sparsified tensor within a model TODO: Need a clean way of loading the state of the "prepared" module cSs"g|] }ttdd|qS)cSs |dtvS)Nr)KEYS_NOT_IN_STATE_DICT) key_valuerrres z6BaseSparsifier.state_dict...)rfilteritems).0mgrrr bsz-BaseSparsifier.state_dict..rr)rr)rrrrr state_dictWs   zBaseSparsifier.state_dictTr8strictcCst|d}|d}|D]g\}}t|j|}|d}|d} |r/|dur/td|dd} |j| D] } t| trAd} nq6| sVtt t || j } t || | |d ddurf|d } | | _|D] } | d |kru| |qhq|||d dS) NrrrrzError loading z into the modelFTmask tensor_fqnr7)copydeepcopyr3r model RuntimeErrorZparametrizations isinstancer torchZonesgetattrshaperregister_parametrizationgetpopr:r#r$)rr8r9rZstatesr;sZarg_inforrfoundpr:r5rrrload_state_dictqs4      zBaseSparsifier.load_state_dictr>SUPPORTED_MODULEScCszg|_|g}|r;|}|D]&\}}t||vr1t||}t|ts&J|jd|diq||q|sdSdS)Nr;z.weight)configrFnamed_childrentyper r@strappend)rr>rKstackr_namechildrrrrmake_config_from_models    z%BaseSparsifier.make_config_from_modelcCs||_||_|jdur|||jD]e}t|tsJdt|jts&Jt|j}||| dd}|dus?Jdt ||}| D]$}||vrl||||ksl|dkrdd||||kslJd|dqH|||j |q|dS)zPrepares a model, by adding the parametrizations. Note:: The model is modified inplace. If you need to preserve the original model, use copy.deepcopy. Nznconfig elements should be dicts not modules i.e.:[{`tensor_fqn`: `foo.bar.weight`}, {`tensor_fqn`: ... }, ...]r;zttensor_fqn is a required argument in the sparsity config whichreplaces previous `module` and [module]`fqn` arguments.z Given both `z?` and `tensor_fqn` in the config, it is expected them to agree!)r>rLrTr@rrr<r=r#rEr r*rrP_prepare)rr>rLZ module_configZ local_argsr;Zinfo_from_tensor_fqnr-rrrprepares6             zBaseSparsifier.preparec Osh|jD].}|d}|d}|dt}|dtt||}||j|dd<t||||qdS)z-Adds mask parametrization to the layer weightrrparametrizationr:r;N) rrEr rAZ ones_likerBrrrD)rargskwargsrLrrrXr:rrrrVs   zBaseSparsifier._prepareparams_to_keep.params_to_keep_per_layerc s|jD]Id}d}tj||ddi}|dur)fdd|D}|||durG|dd} | durGfd d| D} || |rL||_qdS) a= Squashes the sparse masks into the appropriate tensors. If either the `params_to_keep` or `params_to_keep_per_layer` is set, the module will have a `sparse_params` dict attached to it. Args: params_to_keep: List of keys to save in the module or a dict representing the modules and keys that will have sparsity parameters saved params_to_keep_per_layer: Dict to specify the params that should be saved for specific layers. The keys in the dict should be the module fqn, while the values should be a list of strings with the names of the variables to save in the `sparse_params` Examples: >>> # xdoctest: +SKIP("locals are undefined") >>> # Don't save any sparse params >>> sparsifier.squash_mask() >>> hasattr(model.submodule1, 'sparse_params') False >>> # Keep sparse params per layer >>> sparsifier.squash_mask( ... params_to_keep_per_layer={ ... 'submodule1.linear1': ('foo', 'bar'), ... 'submodule2.linear42': ('baz',) ... }) >>> print(model.submodule1.linear1.sparse_params) {'foo': 42, 'bar': 24} >>> print(model.submodule2.linear42.sparse_params) {'baz': 0.1} >>> # Keep sparse params for all layers >>> sparsifier.squash_mask(params_to_keep=('foo', 'bar')) >>> print(model.submodule1.linear1.sparse_params) {'foo': 42, 'bar': 24} >>> print(model.submodule2.linear42.sparse_params) {'foo': 42, 'bar': 24} >>> # Keep some sparse params for all layers, and specific ones for >>> # some other layers >>> sparsifier.squash_mask( ... params_to_keep=('foo', 'bar'), ... params_to_keep_per_layer={ ... 'submodule2.linear42': ('baz',) ... }) >>> print(model.submodule1.linear1.sparse_params) {'foo': 42, 'bar': 24} >>> print(model.submodule2.linear42.sparse_params) {'foo': 42, 'bar': 24, 'baz': 0.1} rrT)Zleave_parametrizedNci|]}||qSrrr4krLrr z.BaseSparsifier.squash_mask..rcr]rrr^r`rrra$rb)rrZremove_parametrizationsr#rE sparse_params) rr[r\rYrZrrrcZ global_paramsparamsZper_layer_paramsrr`r squash_masks& ;  zBaseSparsifier.squash_maskFrmappinginplaceparameterizationc Cs|durtd|st|}i}|D]"\}}t||r,t||vr,t||||<q|j||d|d||<q|D] \}} | |j |<q<|S)aConverts submodules in input module to a different module according to `mapping` by calling `from_dense` method on the target module class Args: module: input module mapping: a dictionary that maps from source module type to target module type, can be overwritten to allow swapping user defined Modules inplace: carry out model transformations in-place, the original module is mutated NzNeed to auto generate mapping T)rfrgrh) NotImplementedErrorr<r=rMr rr convertr3Z_modules) rrrfrgrhZreassignnamemodr-valuerrrrj*s&    zBaseSparsifier.convertuse_pathcCsT|jsdSt|jD] }|jdi|q WddS1s#wYdS)Nr)rrAZno_gradr update_mask)rrnrLrrrstepVs  "zBaseSparsifier.steprcKsdSr!r)rrrrZrrrro]szBaseSparsifier.update_maskr!)T)NN)!r' __module__ __qualname____doc__rrrOrrr r$r.r8boolrJrKrModulesetrNLinearrTrWrVtuplerer rjrpabcabstractmethodro __classcell__rrrrrsP "  3 Q ,)ryr< collectionsrtypingrrrArZtorch.nn.utilsrZtorch.nn.utils.parametrizerutilsr r r r r __all__rwrKr/ABCrrrrrs