o _Zh @sUddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZer^ddlZddlmZedZnedZed Zed Z ee!e!ge"fZ#ee$e!e!fge"fZ%Gd d d Z&Gd dde&Z'Gddde&Z(Gddde&Z)Gddde&Z*Gddde&Z+Gddde&Z,Gddde&Z-GdddeeefZ.Gddde&Z/de&iZ0de1d <d*d+d(d)Z2e&e'e+e(e*e)e,e-e/f D]Z3e2e3qdS),) annotationsN)FutureThreadPoolExecutor)groupby) itemgetter) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleOptional OrderedDictTypeVar) ParamSpecPTZfsspecc@sPeZdZUdZdZded<dd d ZdddZdddZdddZ dddZ dS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]name blocksizeintfetcherFetchersizereturnNonecCs.||_d|_||_||_d|_d|_d|_dSNr)rnblocksrr hit_count miss_counttotal_requested_bytesselfrrrr%=/var/www/auris/lib/python3.10/site-packages/fsspec/caching.py__init__=s zBaseCache.__init__start int | NonestopbytescCs<|durd}|dur |j}||jks||krdS|||S)Nr)rrr$r(r*r%r%r&_fetchGs zBaseCache._fetchcCsd|_d|_d|_dS)zAReset hit and miss counts for a more ganular report e.g. by file.rN)r r!r"r$r%r%r& _reset_statsPs zBaseCache._reset_statsstrc Cs>|jdkr |jdkr dSd|jd|jd|jd|jd S)z2Return a formatted string of the cache statistics.rz , z: z hits, z misses, z total requested bytes)r r!rr"r/r%r%r& _log_statsVs&zBaseCache._log_statscCs@d|jjd|jd|jd|jd|jd|jd|jdS) Nz ) __class____name__rrrr r!r"r/r%r%r&__repr__]szBaseCache.__repr__Nrrrrrrrrr(r)r*r)rr+rr)rr1) r5 __module__ __qualname____doc__r__annotations__r'r.r0r3r6r%r%r%r&r+s    rcsVeZdZdZdZ   d"d#fdd Zd$ddZd%ddZd&ddZd'd d!Z Z S)( MMapCacheamemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: Fetcher Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is location: str Where to create the temporary file. If None, a temporary file is created using tempfile.TemporaryFile(). blocks: set[int] Set of block numbers that have already been fetched. If None, an empty set is created. multi_fetcher: MultiFetcher Function of the form f([(start, end)]) which gets bytes from remote as specified. This function is used to fetch multiple blocks at once. If not specified, the fetcher function is used instead. mmapNrrrrrlocation str | Noneblocksset[int] | None multi_fetcherMultiFetcher | Nonerrcs>t||||durtn||_||_||_||_dSN)superr'setrBr@rD _makefilecache)r$rrrr@rBrDr4r%r&r's  zMMapCache.__init__mmap.mmap | bytearraycCsddl}ddl}|jdkrtS|jdustj|jsB|jdur*|}t |_ nt |jd}| |jd| d|nt |jd}|||jS)Nrzwb+1zr+b)r?tempfiler bytearrayr@ospathexists TemporaryFilerHrBopenseekwriteflushfileno)r$r?rOfdr%r%r&rIs       zMMapCache._makefiler(r)endr+c std|d||durd}|durj}|jks!||kr#dS|j}|j}t||d}fdd|D}jtfdd|D7_g}tt|d d d D]W\}} t t t d| } | dj} t | d jjj} j | | 7_ td | dd| d d| d| d || | fj| jt| 7_qY|sj||Sjrtd |t|D]\} } || \} } td| d| | j| | <qn|D]\} } td| d| | | j| | <qj||S)NzMMap cache fetching -rr,rMc3s|] }|jvr|VqdSrFrB.0ir/r%r& z#MMapCache._fetch..c3s|] }|jvrdVqdS)rMNr]r^r/r%r&rarbcSs|d|dS)NrrMr%)xr%r%r&sz"MMapCache._fetch..)keyzMMap get blocks z ()zMMap copy block (zMMap get block ()loggerdebugrrranger sumr enumeratetuplemaprminr"appendrBupdater!lenrJrDr)r$r(r[Z start_blockZ end_blockZ block_rangeZneedranges_Z_blocksZsstartsendidxrr%r/r&r.sJ    $   zMMapCache._fetchdict[str, Any]cCs|j}|d=|S)NrJ)__dict__copyr$stater%r%r& __getstate__s zMMapCache.__getstate__r|cCs|j|||_dSrF)ryrqrIrJr{r%r%r& __setstate__s zMMapCache.__setstate__)NNN)rrrrrrr@rArBrCrDrErr)rrLr(r)r[r)rr+rrxr|rxrr) r5r:r;r<rr'rIr.r}r~ __classcell__r%r%rKr&r>js   >r>c0eZdZdZdZdfd d ZdddZZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). Z readaheadrrrrrrrcs&t|||d|_d|_d|_dS)Nr,r)rGr'rJr(r[r#rKr%r&r's zReadAheadCache.__init__r(r)r[r+cCs>|durd}|dus||jkr|j}||jks||krdS||}||jkr?||jkr?|jd7_|j||j||jS|j|krK|jkrhnn|jd7_|j||jd}|t|8}|j}n |jd7_d}t|j||j}|j ||7_ | |||_||_|jt|j|_||jd|SNrr,rM) rr(r[r rJr!rrrorr"r)r$r(r[lpartr%r%r&r.s. zReadAheadCache._fetchr7rr5r:r;r<rr'r.rr%r%rKr&rs rcr)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrrrrcs&||kr|}t|||d|_dSrF)rGr'rJr#rKr%r&r'/s zFirstChunkCache.__init__r(r)r[r+cCs*|pd}||jkrtddSt||j}||jkr|jdurW|jd7_||jkrG|j|7_|d|}|d|j|_||dS|d|j|_|j|j7_|j||}||jkrv|j||j7_|||j|7}|j d7_ |S|jd7_|j||7_|||S)Nrz,FirstChunkCache: requested start > file sizer,rM) rrhrirorrJr!r"rr )r$r(r[datarr%r%r&r.6s0          zFirstChunkCache._fetchr7rrr%r%rKr&r&s rcsheZdZdZdZ d#d$fd d ZddZd%ddZd&ddZd'ddZ d(fdd Z d)d!d"Z Z S)* BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z blockcache rrrrr maxblocksrrcs<t|||t|||_||_t||j|_ dSrF) rGr'mathceilrr functools lru_cache _fetch_block_fetch_block_cachedr$rrrrrKr%r&r'mszBlockCache.__init__cC |jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor/r%r%r&ru zBlockCache.cache_inforxcCs|j}|d=|S)Nrryr{r%r%r&r}szBlockCache.__getstate__r|cCs&|j|t|d|j|_dS)Nr)ryrqrrrrr{r%r%r&r~s   zBlockCache.__setstate__r(r)r[r+cCst|durd}|dur |j}||jks||krdS||j}||j}t||dD]}||q)|j||||dS)Nrr,rMstart_block_numberend_block_number)rrrjr _read_cache)r$r(r[rr block_numberr%r%r&r.s    zBlockCache._fetchrcst||jkrtd|d|jd||j}||j}|j||7_|jd7_td|t||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rgrMzBlockCache fetching block %d) r ValueErrorrr"r!rhinforGr.)r$rr(r[block_contentsrKr%r&rs    zBlockCache._fetch_blockrrc C||j}||j}|jd7_||kr ||}|||S|||dg}|t|jt|d||||d|d|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rMNr,rr rextendrnrjrpjoin r$r(r[rr start_posend_posblockoutr%r%r&rs     zBlockCache._read_cacher rrrrrrrrrrrrr)rrrr+ r(rr[rrrrrrr+) r5r:r;r<rr'rr}r~r.rrrr%r%rKr&rTs  rcsHeZdZUdZdZded< ddfdd ZdddZdddZZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r+rrTrrrrrtrimboolrrcs,t|||d|_d|_d|_||_dS)Nr,)rGr'rJr(r[r)r$rrrrrKr%r&r's  zBytesCache.__init__r(r)r[cCs|durd}|dur |j}||jks||krdS|jdurC||jkrC|jdurC||jkrC||j}|jd7_|j||||S|jrPt|j||j}n|}||ks[||jkr]dS|jdusg||jkr|jdusq||jkr|j||7_|jd7_| |||_||_n|jdusJ|jdusJ|jd7_||jkr|jdus|j||jkr|j||7_| |||_||_nb|j|j|7_| ||j}||_||j|_nG|jdur)||jkr)|j|jkrn4||j|jkr|j||7_| |||_||_n|j||j7_| |j|}|j||_|jt |j|_||j}|j||||}|j rk|j|j|jd}|dkrk|j|j|7_|j|j|d|_|Sr) rr(r[r rJrror"r!rrrr)r$r(r[offsetZbendnewrnumr%r%r&r.sn          zBytesCache._fetchcCs t|jSrF)rrrJr/r%r%r&__len__>s zBytesCache.__len__)T) rrrrrrrrrrr)rr) r5r:r;r<rr=r'r.rrr%r%rKr&rs   IrcsDeZdZUdZdZded<    ddfdd ZdddZZS)AllBytesz!Cache entire contents of the fileallrrNrr)rFetcher | Nonerr bytes | NonerrcsNt||||dur"|jd7_|j|j7_|d|j}||_dS)NrMr)rGr'r!r"rrr)r$rrrrrKr%r&r'Gs  zAllBytes.__init__r(r*r+cCs|jd7_|j||S)NrM)r rr-r%r%r&r.UszAllBytes._fetch)NNNN) rr)rrrr)rrrrr8 r5r:r;r<rr=r'r.rr%r%rKr&rBs  rcsDeZdZUdZdZded<  ddfdd Zdfdd ZZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrNTrrrrrr&Optional[dict[tuple[int, int], bytes]]strictrrtrc st|||||_|rdt|}|dg}||dg} |ddD]3\} } |d\} } | | krH| | f|d<| d|| | f7<q&|| | f| || | fq&tt|| |_ dSi|_ dS)NrrMrf) rGr'rsortedkeyspoprpdictzipr)r$rrrrrrtZ old_offsetsoffsetsrBr(r*Zstart0Zstop0rKr%r&r'ss      zKnownPartsOfAFile.__init__r(r)r*rr+cs:|durd}|dur |j}d}|jD]J\\}}}||kr$|kr^nq||}|||||}|jrC||krA|krZnn|d||t|7}|jd7_|S|}nq|jdurntd||fdt d||fdt d|d ||j ||7_ |j d7_ |t||S) Nrr,rMz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching r\)rritemsrrrr rrwarningswarnrhrir"r!rGr.)r$r(r*rZloc0loc1roffrKr%r&r.s2 zKnownPartsOfAFile._fetch)NT) rrrrrrrrrrrtrr8rr%r%rKr&rZs  rc@sTeZdZdZGdddeZdd d d Zd!ddZd"ddZd#ddZ d$ddZ dS)% UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c@s.eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforhitsmissesmaxsizecurrsizeN)r5r:r;r=r%r%r%r& CacheInfos  rfuncCallable[P, T]max_sizerrrcCs0t|_||_||_d|_d|_t|_ dSr) collectionsr_cache_func _max_size_hits_misses threadingLock_lock)r$rrr%r%r&r's zUpdatableLRU.__init__argsP.argskwargsP.kwargsrcOs|r td||j&||jvr-|j||jd7_|j|WdSWdn1s7wY|j|i|}|j,||j|<|jd7_t|j|j krk|jj ddWd|SWd|S1svwY|S)Nz Got unexpected keyword argument rMFlast) TypeErrorrrr move_to_endrrrrrrpopitem)r$rrresultr%r%r&__call__s.     zUpdatableLRU.__call__rrcGs4|j ||jvWdS1swYdSrF)rr)r$rr%r%r& is_key_cacheds$zUpdatableLRU.is_key_cachedrcGsd|j%||j|<t|j|jkr |jjddWddSWddS1s+wYdS)NFr)rrrrrr)r$rrr%r%r&add_keys "zUpdatableLRU.add_keyrcCsH|j|j|jt|j|j|jdWdS1swYdS)N)rrrr)rrrrrrrrr/r%r%r&rs$zUpdatableLRU.cache_infoN)r)rrrrrr)rrrrrr)rrrr)rrrrrrrr) r5r:r;r<r rr'rrrrr%r%r%r&rs    rcsveZdZUdZdZded< d(d)fdd Zd*ddZd+ddZd,ddZ d-ddZ d.d/fd"d# Z d0d&d'Z Z S)1BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrrrrrrrrrrcsZt|||t|||_||_t|j||_t dd|_ d|_ d|_ t |_dS)NrM max_workers)rGr'rrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrrKr%r&r's zBackgroundBlockCache.__init__rcCrrrr/r%r%r&r$rzBackgroundBlockCache.cache_inforxcCs(|j}|d=|d=|d=|d=|d=|S)Nrrrrrrr{r%r%r&r}/sz!BackgroundBlockCache.__getstate__cCsD|j|t|j|d|_tdd|_d|_d|_t |_ dS)NrrMr) ryrqrrrrrrrrrrr{r%r%r&r~8s  z!BackgroundBlockCache.__setstate__r(r)r[r+c Cs|durd}|dur |j}||jks||krdS||j}||j}d}d}|jM|jduro|jdus6J|jrRtd|j |j |jd|_d|_nt ||jko]|kn}|ro|j}|j}d|_d|_Wdn1sywY|durtd|j | |t ||dD]}||q|d} |j%|jdur| |j kr|j| s| |_|j|j| d|_Wdn1swY|j||||dS)Nrr,z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rMasyncr)rrrrrdonerhrrrrrrjrrrsubmitrr) r$r(r[rrZfetch_future_block_numberZ fetch_futureZ must_joinrZend_block_plus_1r%r%r&r.@sv            zBackgroundBlockCache._fetchsyncrlog_infor1csv||jkrtd|d|jd||j}||j}td|||j||7_|jd7_t||}|S)rrrrgz!BlockCache fetching block (%s) %drM) rrrrhrr"r!rGr.)r$rrr(r[rrKr%r&rs   z!BackgroundBlockCache._fetch_blockrrc Crrrrr%r%r&rs     z BackgroundBlockCache._read_cacherrrrr9r)r)rrrr1rr+r)r5r:r;r<rr=r'rr}r~r.rrrr%r%rKr&rs   Lrz!dict[str | None, type[BaseCache]]cachesFclstype[BaseCache]clobberrrrcCs6|j}|s|tvrtd|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rrr)rrrr%r%r&register_caches  r)F)rrrrrr)4 __future__rrrloggingrrQrrconcurrent.futuresrr itertoolsroperatorrtypingrrr r r r r rrr?Ztyping_extensionsrrr getLoggerrhrr+rlistZ MultiFetcherrr>rrrrrrrrrr=rcr%r%r%r&sb  ,    ?.. ee<Q