o vZhm`@s*UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddlm Z ddl mZmZmZmZmZddlmZddlmZdd lmZmZmZdd lmZmZmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'ddl(m)Z)erddl*m+Z+e,e-Z.dZ/dZ0dZ1gdZ2ee3e4d<ddddddddddde5dee5e fde5dee5dee6deeee5e5fd eeee5e5fd!ee3d"e6d#e3fd$d%Z7Gd&d'd'ej8Z9eeefZ:Gd(d)d)Z;d*e;ddde5de5de5f d+d,Zd3ee:ddde5de5de5d-df d4d5Z?d0e:ddde5de5de5d-df d6d7Z@d3ee:ddde5de5de5d-df d8d9ZAGd:d;d;eZBd0e:d-eBfdd?d-ee:fd@dAZDd>d?dBe3d-ee:fdCdDZEd>d?d-ee:fdEdFZFdGe5d-dfdHdIZGdS)JN)datetime)Path)Lock) TYPE_CHECKINGListOptionalTupleUnion)quote) constants)CommitOperationAdd UploadInfo_fetch_upload_modes)LocalUploadFileMetadataLocalUploadFilePathsget_local_upload_pathsread_upload_metadata)DEFAULT_REVISION REPO_TYPES)DEFAULT_IGNORE_PATTERNSfilter_repo_objectstqdm) _format_size) sha_fileobj)HfApi K) 2rd}iiXiCOMMIT_SIZE_SCALET<)revisionprivateallow_patternsignore_patterns num_workers print_reportprint_report_everyapirrepo_id folder_path repo_typer'r(r)r*r+r,r-csdurtdtvrtdtdurtts-tdd|dur4g}nt|tr<|g}|t 7}|durQt pId} t | dd}j |dd } td | | jtfd d d D||d} fdd| D}tdt|dfddt|ddD}t|fddt|D}|D]}|q| rtdt} tdt|| kr| rtt}rtdnq|D]}|qttddS)zUpload a large folder to the Hub in the most resilient way possible. See [`HfApi.upload_large_folder`] for the full documentation. NzFor large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr T)r/r1r(exist_okzRepo created: c3s&|]}|r|VqdSN)is_file relative_toas_posix).0pathr0S/var/www/auris/lib/python3.10/site-packages/huggingface_hub/_upload_large_folder.py ds$z/upload_large_folder_internal..z**/*)r)r*csg|]}t|qSr;)r)r8relpathr:r;r< hsz0upload_large_folder_internal..zFound z candidate files to uploadcsg|] }|t|jfqSr;)r path_in_repo)r8pathsr:r;r<r?lszRecovering from metadata files)Zdescc s&g|]}tjtddqS))statusr.r/r1r')targetkwargs) threadingThread _worker_jobr8_)r.r/r1r'rBr;r<r?ss z zIs done: exiting main loopzUpload is complete!) ValueErrorrrr expanduserresolveis_dir isinstancestrros cpu_countmaxZ create_repologgerinfor/rgloblenrLargeUploadStatusrangestartprintcurrent_reporttimesleep_print_overwriteis_doneloggingjoin)r.r/r0r1r'r(r)r*r+r,r-Znb_coresrepo_urlZfiltered_paths_listZ paths_listitemsthreadsthreadZlast_report_tsr;)r.r0r/r1r'rBr<upload_large_folder_internal2sn          rfc@s4eZdZeZeZeZeZeZ dS) WorkerJobN) __name__ __module__ __qualname__enumautoSHA256GET_UPLOAD_MODE PREUPLOAD_LFSCOMMITWAITr;r;r;r<rgs  rgc@sfeZdZdZdeefddZdefddZde d ed e dd fd d Z de fddZ de fddZd S)rWzBContains information, queues and tasks for a large upload process.rccCs||_t|_t|_t|_t|_t|_d|_ d|_ d|_ d|_ d|_ d|_t|_d|_t|_|jD]A}|\}}|jdurN|j|q<|jdurZ|j|q<|jdkri|jsi|j|q<|jss|j|q|jtdd|jDWdS1swYdS)Ncs |] \}}|jp |jVqdSr4rrr8rIrr;r;r<r=z,LargeUploadStatus.is_done..)ryallrcrr;r;r<r_s$zLargeUploadStatus.is_done)rhrirj__doc__r JOB_ITEM_TrintrboolfloatrrOr[r_r;r;r;r<rWs! ?rWrBc Cs d}t|}|dur dS|\}}|tjkrn|d}z t||j|Wn+ty.tyQ} zt d| t |j |WYd} ~ nd} ~ ww|j |jd8_Wdn1sgwYn|tjkrz t|||||dWn%tyty} zt d| t WYd} ~ nd} ~ ww|D](}|\} } | jrq| jdkr|j|q| jd kr|j|q|j|q|j |jd8_Wdn1swYn|tjkrP|d}zt|||||d|j|Wn-tyty3} zt d | t |j|WYd} ~ nd} ~ ww|j |jd8_Wdn 1sJwYn|tjkrt} d} z t|||||dWn5typty} z t d | t |D] }|j|qd } WYd} ~ nd} ~ wwt| }|| t|||j t|_|jd8_Wdn 1swYn&|tj krt!t"|j |j#d8_#Wdn 1swYq) a Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. Read `upload_large_folder` docstring for more information on how tasks are prioritized. TNrzFailed to compute sha256: r )r.r/r1r'zFailed to get upload mode: rrZregularzFailed to preupload LFS: zFailed to commit: F)$_determine_next_jobrgrm_compute_sha256rvrKeyboardInterrupt ExceptionrSerror traceback format_excruryrzrn_get_upload_moderrrwrxr{ro_preupload_lfsr|rpr\_commitrrVrr}rqr]WAITING_TIME_IF_NO_TASKSr~)rBr.r/r1r'Znext_jobZjobrcrerIrZstart_tsrrr;r;r<rG s             rGrcCs|j1|jdkr:|jdkr:|jdur:t|jdkr:|jd7_tdtj t |jfWdS|jdkra|jdkra|jd7_tdtj t |jfWdS|j dkr|j d7_ tdtj t|j |fWdS|jdkr|jdkr|jd7_td tjt|jfWdS|jdkr|jdkr|jd7_td tjt|jfWdS|j dkr|j dkr|j d7_ td tj t|j |fWdS|jdkr+|jdkstjs+|jd7_td tjt|jfWdS|jdkrN|jd7_td tjt|jfWdS|j dkrt|j d7_ tdtj t|j |fWdS|jdkr|jdkr|jdurt|jdkr|jd7_tdtj t |jfWdS|jdkr|jdkr|jdkr|j dkr|jdkr|jdkr|j dkr|jdkr|jd7_tdtj t |jfWdStdd|jDrtd WddS|jd7_tdtdtjgfWdS1s9wYdS)Nri,r z;Job: commit (more than 5 minutes since last commit attempt)rzJob: commit (>100 files ready)rz&Job: get upload mode (>10 files ready)z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSz Job: sha256zJob: get upload moder&z-Job: commit (1 min since last commit attempt)z Job: commitcsrr4rrr;r;r<r=rz&_determine_next_job..z.All files have been processed! Exiting worker.zNo task available, waiting... (zs))ryr}rxqsizerr\rSrrgrp_get_items_to_commitrvr{rn_get_nrrwr|ro_get_onerurzrmr ZHF_HUB_ENABLE_HF_TRANSFERrrcrTr~rrq)rBr;r;r<rs         % , 4 :  A   M    [ a&rrcCsZ|\}}|jdur&|jd}t||_Wdn1s!wY||dS)z1Compute sha256 of a file and save it in metadata.Nrb)r file_pathopenrhexsave)rrArfr;r;r<rs  rrcc Csfdd|D}t||||t|dddt||D]\}}|\}} |j| _|j| _| |qdS)zmGet upload mode for each file and update metadata. Also receive info if the file should be ignored. cSg|]}t|qSr;_build_hacky_operationr8rr;r;r<r?z$_get_upload_mode..)safe) additionsr1r/headersr'N) rZ_build_hf_headersr zipZ _upload_moderZ_should_ignorerr) rcr.r/r1r'rradditionrArr;r;r<rs  rcCs8|\}}t|}|j||||gdd|_||dS)z'Preupload LFS file and update metadata.)r/r1r'rTN)rZpreupload_lfs_filesrr)rr.r/r1r'rArrr;r;r<rsrcCsDdd|D}|j||||dd|D] \}}d|_||qdS)zCommit files to the repo.cSrr;rrr;r;r<r?&rz_commit..z(Add files using upload-large-folder tool)r/r1r' operationsZcommit_messageTN)Z create_commitrr)rcr.r/r1r'rrArr;r;r<r$s  rc@seZdZdddZdS)HackyCommitOperationAddrNcCs t|jtrt|j|_dSdSr4)rNpath_or_fileobjrrOrr;r;r< __post_init__9s z%HackyCommitOperationAdd.__post_init__)rN)rhrirjrr;r;r;r<r8srcCs|\}}t|j|jd}|jd}|ddd}Wdn1s&wY|jdur4tdtt |j|j |d|_ |S)N)r@rriz&sha256 must have been computed by now!)rrsample) rr@rrpeekrrJrbytesfromhexrZ upload_info)rrArZ operationfilerr;r;r<r>s rrszqueue.Queue[JOB_ITEM_T]cCs |gSr4getrsr;r;r<rNs rncs fddtt|DS)Ncsg|]}qSr;rrHrr;r<r?Srz_get_n..)rXrr)rsrr;rr<rRs rcCsjg}d\}} |dkr|S|tks|tkr|S|}|||\}}|jdkr0|d7}n|d7}q)zXSpecial case for commit job: the number of items to commit depends on the type of files.)rrTrrrr )rMAX_NB_LFS_FILES_PER_COMMITMAX_NB_REGULAR_FILES_PER_COMMITrappendr)rsrcrZ nb_regularrrIrr;r;r<rVs    rreportcs|d7}tjtfdd|D}t|D]}tjdtjdqtj|tjdt |dtj dS) zPrint a report, overwriting the previous lines. Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` to print the report. Note: works well only if no other process is writing to `sys.stdout`! rc3s |] }t|dVqdS)r N)rV)r8lineterminal_widthr;r<r={rz#_print_overwrite..z zrN) shutilget_terminal_sizecolumnssum splitlinesrXsysstdoutwriterVflush)rZnb_linesrIr;rr<r^ns     r^)Hrkr`rPrsrrrEr\rrpathlibrrtypingrrrrr urllib.parser rr Z _commit_apir rrZ _local_folderrrrrrrutilsrrrZutils._cache_managerrZ utils.sharZhf_apir getLoggerrhrSrrrr%r__annotations__rOrrfEnumrgrrWrGrrrrrrrrrrr^r;r;r;r<s                l w "ll& "&