o Zh9,@s,ddlZddlZddlmZddlmZddlmZmZddl m Z zddl m Z m Z mZmZmZmZmZmZmZmZWneyMZzededZ[wweeZeej   dd ed ed ed eee d eee de!e"e efeeff ddZ#GdddeZ$de d edeefddZ%dS)N)IntEnum)Optional)Graph is_submodule)SACStats) lpDot LpInteger LpMaximize LpMinimize LpProblemLpStatuslpSum LpVariable PULP_CBC_CMDvaluezBPlease install pulp package. See: https://github.com/coin-or/pulp.graph memory_budget world_sizeac_units fsdp_unitsreturnc%stj}d}d}tdt}tdtt|ddt} tdtt|dd} tdtt|d} td tt|d} td tt|d} td tt|d}td tt|d}td d}|rt |}t|D]jd|vr|| dk7}qm|rt|D]t fdd|Dr|| dk7}qt|D]!td|D]}j |dkr|| | |dk7}qqt|D]jdr|| dk7}qt|D]vjds.jdjdkr.t djdt djdt djdt dt djdjd<jd|}jd|}|| || ||| k7}qt|D]9|| | k7}jdroqYjd|}jd|}|| |||| k7}qYt|D]Fjd|}jd|}jd}dg|}t|D]}jj|d}d||<q|| ||t|| k7}qjdd|}t|D]jd|}|| | |||k7}qt|D] ||| k7}q t|D]0tjdD]#}jd |}jd!|}|||| |k7}q'qt|D]6jd"}|||| k7}||||k7}|||||d| k7}qQ|||k7}|t|7}td#d$dd%} || }!|!dkrt d&t|!idd'fSi}"t|D]t| jdkrt| jd(|"jd<qtt|jd)}#t|j|}$|"|#|$fS)*a6 MILP to decide which modules to AC and how much memory to discard. The objective is to minimize recomputation time. The constraint is to ensure peak memory is under budget. Args: graph: graph representation of the model as a module submodule tree where each node is a submodule with memory & runtime stats memory_budget: memory budget in GiB world_size: number of GPUs. In the case of FSDP, world_size will be used to compute the amount of parameter and gradient memory on each rank ac_units: a list of user-specified AC units. fsdp_units: a list of FSDP units. AC units cannot be supermodules of FSDP units. Returns: Dict[str, float]: the optimal SAC solution, mapping from module fqn to the percentage of activation memory to **discard** float: the recomputation time of the optimal SAC solution int: upper bound on the peak memory of the optimal SAC solution. note that value of -1 means that the ILP solver failed to find a solution. di@ZSACyrrrdamrcprctmax_mfqnc3s$|] }t|jdVqdS)r!N)rnodes).0Z fsdp_unitriO/var/www/auris/lib/python3.10/site-packages/torch/distributed/_tools/sac_ilp.py ds  zsac_milp..Zis_leafZ sac_memoryZact_fw_per_modulezFor module {%s}: z.activation memory from memory tracker is {%d},z-activation memory from SAC estimator is {%d}.z!Something is wrong. Please check!z&Overriding the latter with the former.Zact_grad_per_moduleZ act_totalZpos_fw_post_orderindexZparam_per_moduleZ grad_totalZ n_segmentsZslopesZ interceptsZ sac_runtime皙?ZgapRelZ timeLimitmsg$Solver failed to find a solution: %s)lenr"r r rmatrixlistrangersetanyZ ad_matrixloggerwarningZ name2nodeZ fw_post_orderrr rsolveerrorr roundvarValuerZ objective)%rrrrrZ num_nodesMZMEM_MULTIPLIERprobrrrrrrrr Z ac_units_setjZACM_iZIA_iZAG_iZTA_iposZcoeffpZP_1ZTG_isZslopeZ interceptZACT_isolverstatusZ ac_decisionsZrecomputation_timeZpeak_memr&r$r'sac_milp#s             * $    " $    ,       rFc@seZdZdZdZdS) SACDecisionrrN)__name__ __module__ __qualname__ RECOMPUTESAVEr&r&r&r'rGsrG sac_statsc sd|kr dksntd|dt|j}tdt}tdtt|ddt |j r>|j D] }||t j jk7}q0nt|j dd|j ddD]\}}|||k7}qM|jD] }||t jjk7}q_|jD]\}}||kr|||k7}qo||t j jk7}qot|t|j} |t|j| k7}|t|j7}td d dd } || } | dkrtd t| gSfd dt|DS)aB This is adapted from -- https://github.com/facebookresearch/xformers/blob/c6c0ac31f1b08542a0bc27278c6ed10f825f6963/xformers/checkpoint.py#L375 Given the SACStats of a module, including list of operators, their memory, runtimes, and metadata, decide via MILP an optimal set of operators to checkpoint under a given ``memory_budget``. Args: sac_stats: the SACStats object of the module memory_budget: a float between zero and one Returns: List[int]: the decision whether each operator should be saved (1) or recomptued (0). rrz5`memory_budget` must be a float between 0 and 1. Got .zSAC-per-modulexNr/r* r,r.csg|] }t|jqSr&)r<r=)r#r%rOr&r' 'sz?get_optimal_checkpointing_policy_per_module..) ValueErrorr2Z func_namesr r rr3r4r5rZforce_store_randomZrand_opsrGrLrzipZ view_like_opsrKZ inplace_opsmathceilsumZmemoryrZruntimesrr:r8r;r ) rMrZnum_opsr?r%i1i2opZ op_parentZ max_memoryrDrEr&rQr'+get_optimal_checkpointing_policy_per_modules8    &  r[)rNN)&loggingrUenumrtypingrZ"torch.distributed._tools.ilp_utilsrrZ&torch.distributed._tools.sac_estimatorrZpulprrr r r r r rrr ImportErrorerr getLoggerrHr8setLevelINFOfloatintr4strtupledictrFrGr[r&r&r&r'sV   4      :