a
    h'J                     @   s4  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZmZmZmZmZ d dlmZmZmZmZmZ e
eef Ze
eeedf f Zdd	 Zee	e  eed
ddZe
eeedf f edddZ eeeg e!f edddZ"eeeeeee ee ef dddZ#ee ee eedddZ$dd Z%e
eeedf f eeeeedddZ&dd Z'eedd d!d"Z(ed#d$d%Z)d&a*e+ Z,da-d'd( Z.d)d* Z/d+d, Z0d-d. Z1d/d0 Z2d1d2 Z3d3d4 Z4d5d6 Z5e j6d7d8 Z7d9d: Z8d;d< Z9d=d> Z:d?d@ Z;dS )A    N)partial)AnyCallableOptionalUnion)Tensor)_add_batch_dim_remove_batch_dim_vmap_decrement_nesting_vmap_increment_nestingis_batchedtensor)_broadcast_to_and_flattentree_flatten	tree_map_tree_unflattenTreeSpec.c                    s    dt   fdd}|S )Nzvtorch.func.{grad, vjp, jacrev, hessian} don't yet support saved tensor hooks. Please open an issue with your use case.c                     s@   t jj  | i |W  d    S 1 s20    Y  d S N)torchZautogradgraphZdisable_saved_tensors_hooks)argskwargsfmessage C/var/www/auris/lib/python3.9/site-packages/torch/_functorch/vmap.pyfn-   s    z.doesnt_support_saved_tensors_hooks.<locals>.fn)	functoolswraps)r   r   r   r   r   "doesnt_support_saved_tensors_hooks'   s
    r   )flat_in_dims	flat_argsreturnc                    sZ   dd t | |D  t dkr(td rRt fdd D rRtd  d d S )	Nc                 S   s"   g | ]\}}|d ur| |qS r   )size.0in_dimargr   r   r   
<listcomp>9   s   z0_validate_and_get_batch_size.<locals>.<listcomp>r   z/vmap: Expected at least one Tensor to vmap overc                 3   s   | ]}| d  kV  qdS )r   Nr   )r%   r#   Zbatch_sizesr   r   	<genexpr>@       z/_validate_and_get_batch_size.<locals>.<genexpr>zTvmap: Expected all tensors to have the same size in the mapped dimension, got sizes z for the mapped dimension)ziplen
ValueErrorany)r    r!   r   r)   r   _validate_and_get_batch_size6   s    r0   )batched_outputsr"   c                 C   s   t | trt| S dS )N   )
isinstancetupler-   )r1   r   r   r   _num_outputsH   s    
r5   )valuenum_elementserror_message_lambdar"   c                 C   s.   t | ts| f| S t| |kr*t| | S r   )r3   r4   r-   r.   )r6   r7   r8   r   r   r   	_as_tupleR   s
    


r9   )in_dimsr   funcr"   c           	      C   s  t | ts8t | ts8tdt| d|  dt|  dt|dkrXtdt| dt|\}}t| |}|d u rtdt| d|  dt| d  d	| d	t	t
||D ]\}\}}t |ts|d urtdt| d|  d
| dt |tr4t |ts4tdt| d|  d
| dt| d	|d ur||  k s\|| krtdt| d|  d
| d|  d|  d|  d|d ur|dk r||  ||< qt|||||fS )Nvmap(z
, in_dims=zv, ...)(<inputs>): expected `in_dims` to be int or a (potentially nested) tuple matching the structure of inputs, got: .r   z)(<inputs>): got no inputs. Maybe you forgot to add inputs, or you are trying to vmap over a function with no inputs. The latter is unsupported.zb, ...)(<inputs>): in_dims is not compatible with the structure of `inputs`. in_dims has structure r2   z but inputs has structure z, ...)(<inputs>): Got in_dim=zE for an input but in_dim must be either an integer dimension or None.z' for an input but the input is of type zT. We cannot vmap over non-Tensor arguments, please use None as the respective in_dimz> for some input, but that input is a Tensor of dimensionality z  so expected in_dim to satisfy -z <= in_dim < )r3   intr4   r.   	_get_nametyper-   r   r   	enumerater,   r   dimr0   )	r:   r   r;   r!   	args_specr    ir'   r&   r   r   r   _process_batched_inputs\   sl    

(rE   )r    r!   
vmap_levelr"   c                    s"    fddt | |D }t||S )Nc                    s(   g | ] \}}|d u r|n
t || qS r   )r   r$   rF   r   r   r(      s   z*_create_batched_inputs.<locals>.<listcomp>)r,   r   )r    r!   rF   rC   batched_inputsr   rG   r   _create_batched_inputs   s    
rI   c                 C   sp   |d u r6t |tjr2t|r2td|  d|  d|S t |tjsbtd|  d|  dt| dt||||S )Nr<   z	, ...): `z5` can not return a BatchedTensor when out_dim is Nonez%` must only return Tensors, got type z3. Did you mean to set out_dims= to None for output?)r3   r   r   r   r.   r@   r	   )namebatched_outputrF   
batch_sizeout_dimr   r   r   _maybe_remove_batch_dim   s    rN   )r1   out_dimsrF   rL   r;   r"   c           	         s   t | \}fdd}t| tjrnttr:g}qttrVtdkrV}qd u rfg}q|  nt}|d u r|   fddt||D }t	|S )Nc                
      s2   t dt  d dtd  d d	d S )Nr<   , ..., out_dims=z`)(<inputs>): out_dims is not compatible with the structure of `outputs`. out_dims has structure r2   z but outputs has structure r=   )r.   r?   r   r   )r;   rO   output_specr   r   incompatible_error   s    
z+_unwrap_batched.<locals>.incompatible_errorr2   c                    s$   g | ]\}}t t| |qS r   )rN   r?   )r%   rK   rM   )rL   r;   rF   r   r   r(      s   z#_unwrap_batched.<locals>.<listcomp>)
r   r3   r   r   r>   r4   r-   r   r,   r   )	r1   rO   rF   rL   r;   Zflat_batched_outputsrR   flat_out_dimsZflat_outputsr   )rL   r;   rO   rQ   rF   r   _unwrap_batched   s"    

rT   c                 C   s8   t | trd S | d u rd S tdt| d| dd S )Nr<   rP   z): `out_dims` must be an int, None or a python collection of ints representing where in the outputs the vmapped dimension should appear.)r3   r>   r.   r?   )xr;   rO   r   r   r   _check_int_or_none   s    
rV   )rO   r;   r"   c                 C   s&   t | trd S ttt|| d|  d S )N)r;   rO   )r3   r>   r   r   rV   )rO   r;   r   r   r   $_check_out_dims_is_int_or_int_pytree   s    
rW   r;   c                 C   s6   t | dr| jS t| tjr.dt| j dS t| S )N__name__zfunctools.partial(z, ...))hasattrrY   r3   r   r   r?   r;   reprrX   r   r   r   r?      s
    
r?   Fc                     s  t rd S t t r"W d    d S tjdddkr8dsLda W d    d S tjdddaddl	m
   fd	d
} | tjjjj | tjjjj | tjjjj | tjjjj | tjjjj | tjjjj | tjjjj | tjjjj da W d    n1 s0    Y  d S )NZPYTORCH_JIT1TatenZIMPLZFuncTorchBatchedr   decomposition_tablec                    s,   |  v rt |  |   ntd|  d S )Nz!could not find decomposition for )VMAP_DECOMPOSITIONS_LIBimplRuntimeError)decompr^   r   r   #_register_python_decomposition_vmap$  s    zElazy_load_decompositions.<locals>._register_python_decomposition_vmap)DECOMPOSITIONS_LOADEDDECOMPOSITIONS_LOCKosenvirongetr   libraryLibraryr`   Ztorch._decompr_   opsr]   Zmse_loss_backwarddefaultZsmooth_l1_loss_backwardZhuber_loss_backwardZnll_loss_forwardZnll_loss2d_forwardZnll_loss_backwardZnll_loss2d_backwardaddr)rd   r   r^   r   lazy_load_decompositions  s0    
ro   c                 O   sp   t   t||  t||| \}}}	}
|d urTt|	|||}t| |||
||fi |S t| |||	|
||fi |S r   )ro   rW   rE   _get_chunked_inputs_chunked_vmap
_flat_vmap)r;   r:   rO   
randomness
chunk_sizer   r   rL   r    r!   rC   chunks_flat_argsr   r   r   	vmap_impl8  s<    
rv   c                 C   s4   | |  }}|g| }| | }|dkr0| | |S )Nr   )append)Ztotal_elemsrt   Zn_chunkschunk_sizes	remainderr   r   r   get_chunk_sizesZ  s    

rz   c                    sN   |f |d ur&t ||}tt| t fddt| |D }t| }|S )Nc                 3   s6   | ].\}}|d ur |j  |dn|gt  V  qd S N)rB   )Ztensor_splitr-   )r%   tr&   Z
split_idxsr   r   r*   j  s   	z&_get_chunked_inputs.<locals>.<genexpr>)rz   r4   	itertools
accumulater,   )r!   r    rL   rt   rx   Zflat_args_chunksru   r   r}   r   rp   d  s    
	rp   c                 C   sH   g }d }| D ]&}t |\}}|| |d u r|}qtt| }||fS r   )r   rw   listr,   )Zchunks_output_Zflat_chunks_outputarg_specoutputflat_outputZ	arg_specsflat_output_chunksr   r   r   _flatten_chunks_output|  s    
r   c                 C   sX   t | |}t|t|ksJ g }t|D ](\}}|tj|| |d d ||< q*|S r{   )r   r-   rA   rw   r   cat)rO   r   r   rS   r   idxrM   r   r   r   _concat_chunked_outputs  s    

r   c                 K   s   g }|dkrt  nd }|D ]L}	t||	}
|
dkr4q|d urFt | |t| |
||	|||fi | qt|\}}~t|||}t||S )Nsamer   )	r   Zget_rng_stater0   Zset_rng_staterw   rr   r   r   r   )r;   r    ru   rC   rO   rs   r   Zchunks_outputrsr!   rL   r   r   r   r   r   r   rq     s2    

rq   c                 C   s   | dvrt d|  d S )N)errorZ	differentr   zLOnly allowed values for randomness are 'error', 'different', or 'same'. Got )rb   )rs   r   r   r   _check_randomness_arg  s    r   c                 c   s(   zt | |}|V  W t  nt  0 d S r   )r   r
   )rL   rs   rF   r   r   r   vmap_increment_nesting  s    
r   c                 K   sZ   t ||<}t||||}	| |	i |}
t|
|||| W  d    S 1 sL0    Y  d S r   )r   rI   rT   )r;   rL   r    r!   rC   rO   rs   r   rF   rH   r1   r   r   r   rr     s    rr   c                    s    fdd}|S )Nc                     sR   t  4}t| |}|i |}t||W  d    S 1 sD0    Y  d S r   )r   wrap_batchedunwrap_batched)r   r   rF   rH   r1   rL   r;   r:   rs   r   r   inner  s    zrestore_vmap.<locals>.innerr   )r;   r:   rL   rs   r   r   r   r   restore_vmap  s    r   c                 C   s4   t | \}}t||}|d us"J t||||}|S r   )r   r   rI   )r   bdimslevelr!   specZ
flat_bdimsresultr   r   r   r     s
    
r   c                    sR   t | \}}t|dkr | dfS  fdd|D }t| \}}t||t||fS )Nr   r   c                    s0   g | ](}t |tjr$tjj| n|d fqS r   )r3   r   r   Z_CZ
_functorchrT   )r%   r'   r   r   r   r(     s   
z"unwrap_batched.<locals>.<listcomp>)r   r-   r,   r   )r   r   r!   r   r   r   r   r   r   r   r     s    
r   )<
contextlibr   r~   rg   	threadingr   typingr   r   r   r   r   r   Ztorch._C._functorchr   r	   r
   r   r   Ztorch.utils._pytreer   r   r   r   r   r>   r4   Z	in_dims_tZ
out_dims_tr   r   r0   r5   strr9   rE   rI   rN   rT   rV   rW   r?   re   Lockrf   r`   ro   rv   rz   rp   r   r   rq   r   contextmanagerr   rr   r   r   r   r   r   r   r   <module>	   sl   	 ?*,"
2

