o ZhCT@sdZddlZddlZddlmZddlZddlmZddgZGdddejj Z Gdd d ejj Z Gd d d ejj Z Gd ddejj Z dS) z We will recreate all the RNN modules as we require the modules to be decomposed into its building blocks to be able to observe. N)Optional)TensorLSTMCellLSTMc seZdZdZejjZdgZ   dddde de d e d dffd d Z dd e de ee e fd ee e ffddZ dde de d ee e ffddZddZedddZedddZZS) raA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` `split_gates`: specify True to compute the input/forget/cell/output gates separately to avoid an intermediate tensor which is subsequently chunk'd. This optimization can be beneficial for on-device inference latency. This flag is cascaded down from the parent classes. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(6, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) split_gatesTNFr input_dim hidden_dimbiasreturnc s||d}t||_||_||_||_|sAtjj|d|fd|i||_ tjj|d|fd|i||_ tj jj |_nAtj|_ tj|_ tj|_dD],}tjj||fd|i||j |<tjj||fd|i||j |<tj jj |j|<qUtj|_tj|_tj|_tj|_tj jj |_tj jj |_tj jj |_tj jj |_d|_d|_tj|_tj|_dS)Ndevicedtyper )inputforgetcelloutput)?r)super__init__ input_size hidden_sizer rtorchnnZLinearigateshgatesaoZ quantizedZFloatFunctionalgatesZ ModuleDictZSigmoid input_gate forget_gateZTanh cell_gate output_gatefgate_cx igate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsZquint8hidden_state_dtypecell_state_dtype) selfrr r r rrfactory_kwargsg __class__R/var/www/auris/lib/python3.10/site-packages/torch/ao/nn/quantizable/modules/rnn.pyr,sj           zLSTMCell.__init__xhiddencCsr|dus|ddus|ddur||jd|j}|\}}|jsQ||}||}|j||}|dd\}} } } | |}| | } | | } | | } nAi} t |j|j|jD]\\} }}}|||||| | <qb| | d}| | d} | | d} | | d} |j| |}|j|| }|j||}|}t|}|j| |}||fS)Nrrrrrr)initialize_hiddenshape is_quantizedrrrraddchunkrr r!r"zipitemsvaluesr#mulr$r%rtanhr&)r+r2r3hxcxrrrrr r!Zout_gategatekeyr#r$r%cyZtanh_cyhyr0r0r1forwardfs<        zLSTMCell.forward batch_sizer7c Csjt||jft||jf}}|r1|j\}}|j\}}tj||||jd}tj||||jd}||fS)NscaleZ zero_pointr)rzerosrr'r(quantize_per_tensorr)r*) r+rFr7hcZh_scaleZh_zpZc_scaleZc_zpr0r0r1r5s    zLSTMCell.initialize_hiddencCdS)NZQuantizableLSTMCellr0r+r0r0r1 _get_namezLSTMCell._get_namecCs(|du|duks J|jd}|jd}||||du|d}|sJtj||j_|dur4tj||j_tj||j_|durHtj||j_|St||g||g|j|jgD]9\} } } t| j ddd| D] \} } tj| | _qi| durt| j ddd| D] \}} tj|| _qqX|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nr4)rr r rrr)dim) r6rr Parameterrweightr rr:r9r<)clswiwhbibhrrrrwbrZw_chunkrAZb_chunkr0r0r1 from_paramss2   &  zLSTMCell.from_paramscCst||jks Jt|dsJd|j|j|j|j|j|d}|j|_|j|j _|j|j _|rH|j D]}|j|_q5|j D]}|j|_qA|S)Nqconfigz$The float module must have 'qconfig'r) type _FLOAT_MODULEhasattrr[Z weight_ihZ weight_hhZbias_ihZbias_hhr\rrr<)rTotherZuse_precomputed_fake_quantrobservedr-r0r0r1 from_floats$    zLSTMCell.from_floatTNNN)F)NNF)FF)__name__ __module__ __qualname____doc__rrrr^Z __constants__intboolrrrtuplerEr5rO classmethodr[rb __classcell__r0r0r.r1rsN ;  ,   $c sleZdZdZ   ddddededed dffd d Zdd ed ee eeffddZ e ddZ Z S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. TNFrrr r r cs2||d}tt||f||d||_dSNr )r r)rrrr)r+rr r r rrr,r.r0r1rs  z_LSTMSingleLayer.__init__r2r3cCsNg}|jd}t|D]}||||}||dq t|d}||fS)Nr)r6rangerappendrstack)r+r2r3resultZseq_leniZ result_tensorr0r0r1rEs   z_LSTMSingleLayer.forwardcOs2tj|i|}||j|j|j|jd}||_|S)Nr)rr[rrr rr)rTargskwargsrlayerr0r0r1r[s z_LSTMSingleLayer.from_paramsrcrd)rerfrgrhrirjrrrrkrErlr[rmr0r0r.r1rns&    rncszeZdZdZ     ddddededed ed ed df fd d Zddedee eeffddZ e dddZ Z S) _LSTMLayerz#A single bi-directional LSTM layer.TFNrrr r batch_first bidirectionalr c sb||d} t||_||_t||f||d| |_|jr/t||f||d| |_dSdSro)rrryrzrnlayer_fwlayer_bw) r+rr r ryrzr rrr,r.r0r1rs,  z_LSTMLayer.__init__r2r3cCs|jr |dd}|durd\}}n|\}}d}|jrE|dur"d}n|d}|d}|dur1d}n|d}|d}|durE|durE||f}|durP|durPd}n tj|tj|f}|||\} }t|dr|jr|d} | | |\} }| d} t | | g| d} |dur|durd} d}n=|durtj|\} }n0|durtj|\} }n#t |d|dgd} t |d|dgd}n | } tj|\} }|jr| dd| | |ffS)Nrr4)NNr|)ry transposerzrjit_unwrap_optionalr{r_flipr|catrQrrZ transpose_)r+r2r3Zhx_fwZcx_fwZ hidden_bwZhx_bwZcx_bwZ hidden_fwZ result_fwZ x_reversedZ result_bwrsrKrLr0r0r1rE*sT      z_LSTMLayer.forwardrcKsPt|ds |dus J|d|j}|d|j}|d|j}|d|j}|d|j} |dd } |||||| | d } t|d|| _t|d |} t|d |} t|d |d}t|d|d}t j | | ||| d | _ |jrt|d |d} t|d |d} t|d |dd}t|d|dd}t j | | ||| d | _ | S)z There is no FP equivalent of this class. This function is here just to mimic the behavior of the `prepare` within the `torch.ao.quantization` flow. r\Nrrr ryrzrFrZ weight_ih_lZ weight_hh_lZ bias_ih_lZ bias_hh_lZ_reverse) r_getrrr ryrzgetattrr\rnr[r{r|)rTr`Z layer_idxr\rvrrr ryrzrrwrUrVrWrXr0r0r1rb`s@   z_LSTMLayer.from_float)TFFNNrd)rN)rerfrgrhrirjrrrrkrErlrbrmr0r0r.r1rx s2   6rxcseZdZdZejjZ       dddded ed ed e d e d e de de ddffddZ dde de ee e ffddZddZedddZeddZZS) raXA quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> # xdoctest: +SKIP >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer r4TFNrrr num_layersr rydropoutrzrr c  s|| dt|_|_|_|_|_t|_|_ d_ t |t j r>d|kr4dkr>ntdt |trBtd|dkrZtd|dkrZtd|d|tjjjfdj d g} | fd d td|Dtj| _dS) Nr Frr4zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=ryrzrc3s4|]}tjjjfdjdVqdS)FrN)rxrr rz.0_r,r+rr0r1 s  z LSTM.__init__..)rrrrrr ryfloatrrztraining isinstancenumbersNumberrj ValueErrorwarningswarnrxextendrprrZ ModuleListlayers) r+rrrr ryrrzr rrrr.rr1rsb       z LSTM.__init__r2r3cs|jr |dd}|d}|jrdnd}|durEtj|||jtj|jd d|j r8tj dd|j dfddt |jD}n5tj|}t|dtrx|d|j|||j|d|j|||jfd dt |jD}n|}g}g}t|jD]!\} } | ||| \}\} } |tj| |tj| qt|} t|}| d | jd | jd } |d |jd |jd }|jr|dd}|| |ffS) Nrr4)rr rrGcsg|]}fqSr0r0r)rIr0r1 sz LSTM.forward..cs(g|]}|d|dfqS)r)Zsqueeze)ridx)r@r?r0r1rs)ryr}sizerzrrIrrr Zsqueeze_r7rJrrprr~rrrZreshape enumeraterrqrrr6)r+r2r3Zmax_batch_sizeZnum_directionsZhxcxZhidden_non_optZhx_listZcx_listrrwrKrLZ hx_tensorZ cx_tensorr0)r@r?rIr1rEsV            z LSTM.forwardcCrM)NZQuantizableLSTMr0rNr0r0r1rO+rPzLSTM._get_namec Cst||jsJt|ds|sJ||j|j|j|j|j|j|j |d}t |d||_ t |jD]}t j|||d|d|j|<q0|jrR|tjjj|dd}|S|tjjj|dd}|S)Nr\rF)ryrT)Zinplace)rr^r_rrrr ryrrzrr\rprxrbrrtrainrrZ quantizationZ prepare_qatevalprepare)rTr`r\rrarr0r0r1rb.s0  zLSTM.from_floatcCstd)NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rTr`r0r0r1 from_observedKszLSTM.from_observed)r4TFrFNNrd)NF)rerfrgrhrrrr^rirjrrrrrkrErOrlrbrrmr0r0r.r1rsJ    J7 )rhrrtypingrrr__all__rModulerrnrxrr0r0r0r1s  O*