o
    Zhu                     @   s   d dl mZ d dlmZmZmZ d dlmZmZ d dl	Z	d dl
mZ d dlmZmZ ddlmZmZ eddejd	ee fd
dZG dd deZeedejdejfddZdS )    )	Generator)AbstractContextManagercontextmanagernullcontext)AnyOptionalN)'_checkpoint_without_reentrant_generator_DEFAULT_DETERMINISM_MODE   )_Statecontractmoduleuser_ctxc                 c   st    |r|nt  ( t| j}dt| _zdV  W |t| _n|t| _w W d   dS 1 s3w   Y  dS )zs
    Disable hooks installed by checkpoint to avoid unintentional recursion
    during backward recomputation.
    FN)r   
checkpointstateenable_hook)r   r   Zorig_enable_hook r   b/var/www/auris/lib/python3.10/site-packages/torch/distributed/_composable/checkpoint_activation.py_no_hook   s   "r   c                   @   s*   e Zd ZU dZeed< eed  ed< dS )_CheckpointStateFr   )NNN_ac_generatorN)__name__
__module____qualname__r   bool__annotations__r   r   r   r   r   r   r       s   
 r   returnc                    s   t jd |dd}|rtd|dd|dd|d	t|d
d |r:tdddd |D  dtj	dt
tdf dtttf ddf fdd}dtj	dt
tdf dtdtfdd}dt| _| j|dd | j|ddd | S )a  
    This is a composable activation checkpointing API. Unlike functional
    activation checkpointing APIs, this one does not require changing model
    source code. Unlike ``nn.Module`` wrapper activation checkpointing APIs,
    this one does not modify model structure or fully-qualified names either.
    Under the hood, it registers activation checkpointing logic as pre- and
    post-forward hooks. Hence, this API can be easily applied to any model or
    sub-modules in the model.

    Args:
        module (nn.Module): the target model or sub-module to apply activation
            checkpointing.

    Example::
        >>> # xdoctest: +SKIP
        >>> import torch.nn as nn
        >>>
        >>> class MyModel(nn.Module):
        >>>     def __init__(self) -> None:
        >>>         super().__init__()
        >>>         self.l1 = nn.Linear(10, 10)
        >>>         self.l2 = nn.Linear(10, 10)
        >>>
        >>>     def forward(self, x):
        >>>         return self.l2(self.l1(x))
        >>>
        >>> model = MyModel()
        >>> checkpoint(model.l1)  # apply activation checkpointing only to l1
        >>> model(torch.zeros(2, 10)).sum().backward()

    ztorch.distributed.checkpointuse_reentrantFzsuse_reentrant=True is not supported in composable checkpoint. Please use torch.utils.checkpoint.checkpoint instead.preserve_rng_stateTZ
context_fnNdeterminism_checkdebugzUnexpected keyword arguments: ,c                 s   s    | ]}|V  qd S Nr   ).0argr   r   r   	<genexpr>U   s    zcheckpoint.<locals>.<genexpr>r   args.kwargsr   c                    sT   t  jr( fdd}t |g|R i |}|t  _t| d S d S )Nc                     s.   d ur \} }| t  |fS t t  fS r"   )r   r   )Zctx1Zctx2)r   user_context_fnsr   r   context_fns]   s   
z9checkpoint.<locals>.forward_pre_hook.<locals>.context_fns)r   r   r   r   r   next)r   r&   r'   r)   genr    r   r   r(   )r   r   forward_pre_hookX   s    	z$checkpoint.<locals>.forward_pre_hookinputsoutputc                 S   sZ   t | jr%zt | j}|d usJ t| W td ty$   Y nw d t | _d S )NzWExpected non-reentrant activation checkpoint generator to be exhausted, but it was not!)r   r   r   r   r*   StopIterationRuntimeError)r   r.   r/   r+   r   r   r   forward_hookp   s   
	z checkpoint.<locals>.forward_hook)Zwith_kwargs)prependZalways_call)torchZ_CZ_log_api_usage_oncepopNotImplementedErrorr	   
ValueErrorjoinnnModuletupler   dictstrr   r   r   Zregister_forward_pre_hookZregister_forward_hook)r   r'   r   r-   r2   r   r,   r   r   %   s8   !

$r   r"   )collections.abcr   
contextlibr   r   r   typingr   r   r4   Ztorch.nnr9   Ztorch.utils.checkpointr   r	   r   r   r:   r   r   r   r   r   r   r   <module>   s   