
    7ThӪ                        S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
Jr  S SKrS SKJr  S SKJrJrJrJrJrJr  S SKJr  S SKJr  S SKJrJr  S SKJr  S SK J!r!J"r"J#r#J$r$J%r%  S S	K&J'r'J(r(J)r)  S
 r*S r+S r,S r-S r.\%\$S 5       5       r/\%\$\)" \R                  S:g  SS9S 5       5       5       r0\%\$S 5       5       r1\%S 5       r2\%\$\'" S\\Rf                  /SS/S9S 5       5       5       r4\%\$\'" S\\Rf                  /SS/S9S 5       5       5       r5\%\$\'" SSS/5      S 5       5       5       r6\%\$\'" SSS/5      S 5       5       5       r7\%\$\'" SSS/5      S 5       5       5       r8\%\$\'" SSS/5      S  5       5       5       r9\%\$S! 5       5       r:\%\$S" 5       5       r;\%\$\'" SSS/5      S# 5       5       5       r<\%\$\'" SSS/5      S$ 5       5       5       r=\%\$\'" SSS/5      S% 5       5       5       r>\%\$\'" S\\Rf                  /SS/S9S& 5       5       5       r?\%\$\'" S\R                  " S\R                  R                  S'S9S(9S/5      S) 5       5       5       rC\%\$\'" S\\Rf                  /SS/S9S* 5       5       5       rD\%\$\#\'" S\\Rf                  /SS/S9S+ 5       5       5       5       rE\%\$\#\'" S\\Rf                  /SS/S9S, 5       5       5       5       rF\%\$\'" S\\Rf                  /SS/S9S- 5       5       5       rGS. rH\%\$\'" S\\Rf                  /SS/S9S/ 5       5       5       rIS0 rJ\%\$\'" S1\K" \R                  " \\Rf                  /\M" S25      5      5      \R                  " SS/\N" \O\M" S25      5      5       V Vs/ s H  u  pS3R                  X5      PM     snn S9S4 5       5       5       rQS5 rRS6 rS\%\)" \R                  S:H  S7S9\'" S8S9S:/5      S; 5       5       5       rT\%S< 5       rUS= rV\%\$S> 5       5       rW\%\$S? 5       5       rXgs  snn f )@    N)sleep)ArrayMemmapForwardReducer_get_backing_memmap_get_temp_dir_strided_from_memmap_WeakArrayKeyMaphas_shareable_memory)make_memmap)_TestingMemmappingExecutor)Paralleldelayed)MemmappingPool)IS_GIL_DISABLEDnpwith_dev_shmwith_multiprocessing
with_numpy)parametrizeraisesskipifc                  .    [         R                  " SSS9  g )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     S/var/www/auris/envauris/lib/python3.13/site-packages/joblib/test/test_memmapping.pysetup_moduler   &   s    %%c=r   c                  .    [         R                  " 5         g N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler#   *   s    ,,.r   c                 "    [        U 5      c   eU $ r!   )r   )arrays    r   check_memmap_and_send_backr&   .   s    u%111Lr   c                 R    U u  pn[         R                  R                  X   U5        g)zDummy helper function to be executed in subprocesses

Check that the provided array has the expected values in the provided
range.

Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr0   3   s$      $DHJJ!!$.(;r   c                     U u  pnX   U:X  d   eX==   S-  ss'   [         R                  R                  X   SU-  5        g)zDummy helper function to be executed in subprocesses


Check that the input array has the right values in the provided range
and perform an inplace modification to double the values in the range by
two.

   Nr(   r+   s       r   inplace_doubler3   >   sD      $DH>X%%%NaNJJ!!$.!h,?r   c           	      ^  ^ [         R                  R                  nU R                  S5      R                  n[         R
                  " U[         R                  SSS9nS[         R                  " UR                  S   UR                  S9-  USS& UR                  5         [         R
                  " U[         R                  S	S
SSS9n[         R                  " S5      R                  UR                  5      USS& USS2SS2SS24   nUR                  SS5      n[         R                  " U5      nUR                  n[        SU R                  SS5      mU4S jn	U	" U5      n
[        U
5      (       d   e[!        U
[         R
                  5      (       d   eU" X5        U	" U5      n[        U5      (       d   eU" X5        U	" U5      n[        U5      (       d   eU" X5        U	" U5      n[!        U[         R
                  5      (       a   e[        U5      (       d   eU" X5        U	" U5      n[!        U[         R
                  5      (       a   e[        U5      (       d   eU" X5        US-  n[        U5      (       a   eU	" U5      n[        U5      (       a   e[!        U[         R
                  5      (       a   eU" UUS-  5        [         R                  " U5      n[        U5      (       a   eU	" U5      n[!        U[         R"                  5      (       d   e[        U5      (       a   eU" UU5        g)z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r8   N)         r+Fr>   )r8   r9   r:   orderoffset<      r2   
   2   cTc                 "   > T" U 5      u  pU" U6 $ r!   r   xconsr,   reducers      r   reconstruct_array_or_memmapEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmaps       QZ
T{r   r<   )r   r)   r*   joinstrpathmemmapfloat64aranger9   r8   flushreshapeasarrayTr   r	   
isinstancendarray)tmpdirr*   filenamebufferabb2rH   drN   a_reconstructedb_reconstructedb2_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrM   s                      @r    test_memmap_based_array_reducingrl   M   s    66{{;'//H YYxrzz4HF ryyaEEF1I
LLN 			

)$cRS	A 99R=  )AaD
 	
!B$"ac/A 
B	B 	

1A	A (fnnc4HG
 2!4O0000oryy1111* 2!4O0000* 326 01111', 2!4O/29955550000*1!4O/29955550000* 
QB#B''''226#$45555*BII6666'Q/ 
BB#B''''226&

3333#$45555',r   win32z2PermissionError only easily triggerable on Windows)reasonc                    U R                  S5      R                  nSR                  US9n[        R                  " [
        R                  SU/[        R                  [        R                  S9nUR                  5         UR                  5       u  pEUR                  S:X  d   UR                  5       5       eUS:X  d   eSR                  U5      nXeR                  5       ;   d   eg )	Nr5   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r]   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rQ   rR   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)r\   r]   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;'//H: 	!; < 		s#JOOJOO	A FFH}}HC<<1*cjjl*#::
3
:
:8
DC**,r   c                 >  ^ [         R                  R                  nU R                  S5      R                  n[         R
                  " U[         R                  SSS9n[         R                  " S5      R                  UR                  5      US S & USS nUS S 2SS24   nUS S 2S S 2S S 2S4   nUS	S
S2   n[        S U R                  SS5      mU4S jnU" U5      n	[        U	5      (       d   e[        U	[         R
                  5      (       d   eU" X5        U" U5      n
[        U
5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        g )Nr5   )d      r   r<   r6   r7   i r   rF   r=   rD   r<   r>   rH   Tc                 "   > T" U 5      u  pU" U6 $ r!   r   rJ   s      r   rN   Ntest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rP   r   )r   r)   r*   rQ   rR   rS   rT   rU   rW   r9   r   r	   rZ   )r\   r*   r]   r_   r`   rH   rb   erN   rc   rd   rf   rg   e_reconstructedrM   s                 @r   )test_high_dimension_memmap_array_reducingr      s    66{{;'//H 			("**4D4PA99&'//8AaD 	
!BA	!QrT'
A	!Q1*A	!Aa%A (fnnc4HG 2!4O0000oryy1111*1!4O0000*1!4O0000*1!4O0000*1!4O0000*r   c                    U R                  S5      R                  nS[        R                  -  n[        R                  S-   n[        R
                  " USX#-   S9n[        USSUSUS S S	S
9	n[        U[        R
                  5      (       d   eUR                  U:X  d   e[        USSUSUS-  4SUS	S
9	n[        U5      R                  U:X  d   eg )Nr5   r=   rD   r6   )r:   r9   uint8rCF)r8   r:   rB   rA   r9   stridestotal_buffer_lenunlink_on_gc_collectr2   )r2   )
rQ   rR   mmapALLOCATIONGRANULARITYr   rS   r   rZ   rB   r   )r\   fnamesizerB   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr     s    KK$,,Et)))D''!+F5t4=AJ%"
J j")),,,,&&&,qyl"
 0188FBBBr   factorymultiprocessingloky)idsc                    [         R                  R                  nUR                  S5      R                  nU " SSUS9n UR                  S5      R                  n[         R                  " U[         R                  SSS9nUR                  S	5        UR                  [        [        UR                  S
   5       VVs/ s H(  n[        UR                  S   5        H  oXx4S	4PM
     M*     snn5        U" US[         R                  " UR                  5      -  5        [         R                  " U[         R                  SSS9n	UR                  [        [        U	R                  S
   5       VVs/ s H(  n[        U	R                  S   5        H  oXx4S4PM
     M*     snn5        [        R                  " U5      / :X  d   eU" US[         R                  " UR                  5      -  5        U" U	S[         R                  " U	R                  5      -  5        [         R                  " U[         R                  SSSS9n
[!        ["        5         UR                  [$        [        U
R                  S
   5       Vs/ s H  ozUS4PM	     sn5        SSS5        [!        [&        [(        45         UR                  [        [        U
R                  S
   5       Vs/ s H  ozUS4PM	     sn5        SSS5        UR+                  5         Ags  snnf s  snnf s  snf ! , (       d  f       N= fs  snf ! , (       d  f       NG= f! UR+                  5         Af = f)z@Check that subprocess can access and update shared memory memmappoolrF   r2   
max_nbytestemp_folderr5   r<   r=   r6   r7         ?r   rD   )r=   r<   rH   g       @rF   r      )r8   r9   r:   rB   g      @N)r   r)   r*   mkdirrR   rQ   rS   float32fillmapr3   ranger9   onesoslistdirr   AssertionErrorr0   RuntimeError
ValueError	terminate)r   r\   r*   pool_temp_folderr   r]   r_   ijr`   rH   s              r   test_pool_with_memmapr   (  s    66 ||F+33q.>?A);;{+33IIhbjjTJ	s	',QWWQZ'8T'8!%PQ
BSQ!BS'8T	

 	1a"''!''"223 IIhbjjSI	',QWWQZ'8T'8!%PQ
BSQ!BS'8T	
 zz*+r111 	1a"''!''"2231a"''!''"223 IIhbjjCPUVN#EE+U1771:5FG5FAs5FGH $
 \:./EE.aggaj8I"J8I1q#;8I"JK 0 	
E U U  H $# #K 0/ 	
s   A<L7 ;/K?*BL7 ,/LB-L7 +L3LLL7 %+L&L!L&%L7 ?L7 L
LL7 !L&&
L40L7 7M
c                    [         R                  R                  nUR                  S5      R                  nU " SSUS9n UR                  S5      R                  n[         R                  " U[         R                  SSS9nUR                  S	5        [         R                  " U5      n[        U[         R                  5      (       a   e[        U5      (       d   eUR                  [        [        UR                  S
   5       VV	s/ s H)  n[        UR                  S   5        H	  n	XxU	4S	4PM     M+     sn	n5        U" US[         R                   " UR                  5      -  5        U" US[         R                   " UR                  5      -  5        ["        R$                  " U5      / :X  d   e UR'                  5         Ags  sn	nf ! UR'                  5         Af = f)z?Check that subprocess can access and update shared memory arrayr   rF   r2   r   r5   r   r6   r7   r   r   rD   N)r   r)   r*   r   rR   rQ   rS   r   r   rX   rZ   r	   r   r3   r   r9   r   r   r   r   )
r   r\   r*   r   r   r]   r_   a_viewr   r   s
             r    test_pool_with_memmap_array_viewr   b  s    66 ||F+33q.>?A;;{+33IIhbjjTJ	s Afbii0000#F++++	 qwwqz**Aqwwqz*A Q%* &*	
 	1a"''!''"22361rwwqww'7#78 zz*+r111 	
#  	
s   CG 0F?4A8G ?G Gbackendc                 ^   SR                  U S9n[        R                  " [        R                  SU/[        R
                  [        R
                  S9nUR                  5         UR                  5       u  p4UR                  S:X  d)   UR                  5       S-   UR                  5       -   5       eg )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r`   rp   rq   r   

)
rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r   r~   r   r   r   s        r   -test_permission_error_windows_reference_cycler     s    $ 	% & 		s#JOOJOO	A FFH}}HC<<1BcjjlV3cjjlBBr   c                    SR                  U S9n[        S5       H  n[        R                  R	                  5       n[        R
                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pVUR                  S:X  d   U5       eUS:X  d   eS	U;  a  M   e   g )
Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r<   
PYTHONPATHrp   rr   rs   envr   r      resource_tracker)rt   r   r   environcopypathdirname__file__ru   rv   rw   rx   ry   rz   r{   r|   )r   r~   _r   r   r   r   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s    ( 	) , 1XjjooGGOOH5L^^T3'????	
 	
==?||q %#% czz"#--- r   c                 `  ^ [         R                  " [        S5      5      m[        SU SS9" U4S j[	        S5       5       5      u  n[        SU SS9" U4S j[	        S5       5       5      u  n[
        R                  R                  U5      [
        R                  R                  U5      :w  d   eg )N      Y@r2   rF   n_jobsr   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr]   Nr   getattr.0r   r%   s     r   	<genexpr>6test_parallel_isolated_temp_folders.<locals>.<genexpr>  $      F5=
++X   #&rD   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   r     r   r   r   rU   intr   r   r   r   r   )r   
filename_1
filename_2r%   s      @r   #test_parallel_isolated_temp_foldersr     s     IIc#hE1g"E F5:1XF LZ 1g"E F5:1XF LZ 77??:&"''//**EEEEr   c                   ^ [         R                  " [        S5      5      m[        SU SS9 nU" U4S j[	        S5       5       5      u  nU" U4S j[	        S5       5       5      u  nS S S 5        [
        R                  R                  W5      [
        R                  R                  W5      :X  d   eg ! , (       d  f       NQ= f)Nr   r2   rF   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>        Oh)%<<hr   rD   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   r     r   r   r   )r   r   r   r   r%   s       @r   &test_managed_backend_reuse_temp_folderr     s    
 IIc#hE	G	;qOeAhOOOeAhOO 
< 77??:&"''//**EEEE 
<	;s   ;B22
C c                     [         R                  " [        S5      5      n [        5       n[        5       nS n[        R
                  " X0U4S9n[        R
                  " X0U4S9nUR                  5         UR                  5         UR                  5         UR                  5         [        U5      S:X  d   e[        U5      S:X  d   eX:w  d   eg )Nr   c                 
  ^  [        SSSS9 n[        S5       HN  nU" U 4S j[        S5       5       5      u  nUR                  [        R                  R                  U5      5        MP     S S S 5        g ! , (       d  f       g = f)Nr   r2   rF   )r   r   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   ]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  s      UHqww/zBBHr   rD   )r   r   addr   r   r   )r%   	temp_dirsr   r   r]   s   `    r   concurrent_get_filenameJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  s[    fQ2>!2YUERSHUU
bggooh78  ?>>s   AA44
B)targetr,   rD   )	r   rU   r   set	threadingThreadstartrQ   len)r%   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr      s     IIc#hE9 
		&5G-H
B 
		&5G-H
B HHJHHJGGIGGI!"a'''!"a'''333r   c            
         Sn / SQnU H  u  p#n[         R                  " [        R                  SU R	                  X#S9/[         R
                  [         R
                  S9nUR                  5         UR                  5       u  pgUR                  U:X  d   UR                  5       5       eSU;  a  M   UR                  5       5       e   g )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserD   )r   r   rD   rp   )f1f2rq   r   )
ru   rv   rw   rx   rt   ry   rz   r{   r|   r}   )r~   functions_and_returncodesr   r   r|   r   r   r   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr   &  s    1Cd! 8
^^T3:::#;<????

 	
||z)73::<7)"#-;szz|;- 8r   c                    SR                  U S9n[        R                  R                  5       n[        R                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pEUR                  S:X  d   UR                  5       5       eUS:X  d   UR                  5       5       eSU;  d   eg )	Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rp   r   r   r   r   )rt   r   r   r   r   r   r   ru   rv   rw   rx   ry   rz   r{   r|   r}   )r   r~   r   r   r   r   s         r   'test_many_parallel_calls_on_same_objectr  q  s    $ 	% & **//
C1C	s#		A FFH}}HC<<1*cjjl*#:#szz|#:c)))r   c                    ^ [         R                  " [        S5      5      m[        SU SS9" U4S j[	        S5       5       5      u  n[        U5      b   eg )Ng     @@r2   r   r   c              3   N   >#    U  H  n[        [        5      " T5      v   M     g 7fr!   )r   r&   )r   r   r-   s     r   r   8test_memmap_returned_as_regular_array.<locals>.<genexpr>  s#      C;Ca*+D118s   "%rD   )r   r   r   r   r   r   )r   resultr-   s     @r   %test_memmap_returned_as_regular_arrayr    sR     773s8Dq'cB C;@8C HV v&...r   c                     U S:X  a:  [         R                  R                  S5      (       a  [        R                  " S5        SR                  U S9n[        R                  " [         R                  SU/[        R                  [        R                  S9nUR                  5         UR                  5       u  p4UR                  5       nUR                  5       nUR                  S:X  d   US	-   U-   5       eS
U;  d   U5       eg )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rp   rq   r   r   resource_tracker)rw   platform
startswithpytestxfailrt   ru   rv   rx   ry   rz   r{   r}   r|   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr    s    " &S\\44U;; 	,	

$ 	% & 		s#JOOJOO	A FFH}}HC
**,C
**,C<<10cFlS00S(-#-(r   c           	         [         R                  " UR                  5      / :X  d   eU " SSUR                  SS9n [         R                  " UR                  5      / :X  d   e[         R                  R	                  UR
                  5      (       a   e[        R                  " S[        R                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oCUS	4PM	     sn5        [         R                  " UR                  5      / :X  d   e[        R                  " S
[        R                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oEUS	4PM	     sn5        [         R                  R                  UR
                  5      (       d   e[         R                  " UR
                  5      n[!        U5      S:X  d   e[        R"                  " S/S
-  SS9nUR                  [$        U/5      nUS   (       a   e UR'                  5         [        S5       H?  n[)        S5        [         R                  R	                  UR
                  5      (       a  M>    Ag   [+        SR-                  UR
                  5      5      es  snf s  snf ! UR'                  5         [        S5       H?  n[)        S5        [         R                  R	                  UR
                  5      (       a  M>    Af    [+        SR-                  UR
                  5      5      e= f)z0Check that large arrays are not copied in memoryr<   (   r2   )r   r   verboser=   r;   r   r   r   r      rD   abcobjectrF   皙?z#temporary folder {} was not deletedN)r   r   rR   r   exists_temp_folderr   r   r   nbytesr   r0   r   r9   rT   isdirr   r%   r	   r   r   r   rt   )	r   r\   r   smallr   largedumped_filenamesobjectsresultss	            r   %test_memmapping_pool_for_large_arraysr     s    ::fnn%+++ 	bfnnaHA(zz&..)R///77>>!..1111,||r!!!	kU5;;q>5JK5JAsO5JKL zz&..)R/// 2::.||s"""	kU5;;q>5JK5JAsO5JKL ww}}Q^^,,,,::ann5#$))) ((E7S=9%%,wi81:~: 	
rA#J77>>!..11
  
 !5<<Q^^L = L L  	
rA#J77>>!..11
  
 !5<<Q^^L s3   B5K	 0J?
>BK	 
K
B$K	 ?
K	 	AM#)Mz,https://github.com/joblib/joblib/issues/1086)marksc                 x   SR                  U S9n[        R                  R                  5       n[        R                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pEUR                  5       UR                  5       pTUR                  S5      S   nUR                   S:X  d   U=(       d    U5       eUS:X  d   e[        R                  R#                  U5      (       a   eg )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rp   r   
r    )rt   r   r   r   r   r   r   ru   rv   rw   rx   ry   rz   r{   r}   splitr|   r  )r   r~   r   r   r   r   r]   s          r   &test_child_raises_parent_exits_cleanlyr&    s    :-Z 	[ \ **//
C1C	s#		A FFH}}HCzz|SZZ\yyq!H<<1(cjS("99ww~~h'''''r   c           	         U " SSUR                   S9n [        R                  " UR                   5      / :X  d   e[        R                  " S[        R
                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oCUS4PM	     sn5        [        R                  " UR                   5      / :X  d   e UR                  5         Ags  snf ! UR                  5         Af = f)	z2Check that large arrays memmapping can be disabledr<   Nr   r   r;   r  r   r   )rR   r   r   r   r   rT   r  r   r0   r   r9   r   )r   r\   r   r  r   s        r   .test_memmapping_pool_for_large_arrays_disabledr(  w  s     	d?Azz&..)R/// 2::.||s"""	kU5;;q>5JK5JAsO5JKL zz&..)R/// 	
 L 	
s   BC, C'
'-C, 'C, ,C?c                    [         R                  n [        S5      [         l        U " SSS9n UR                  nSnUR	                  U5      (       d   e[
        R                  R                  U5      (       d   e[        R                  " S[        R                  S9nUR                  S:X  d   eUR                  [        U/S-  5        [        [
        R                  " U5      5      S	:X  d   e[        R                  " S[        R                  S9S
-  nUR                  S:X  d   eUR                  [        U/S-  5        [        [
        R                  " U5      5      S
:X  d   e UR!                  5         A[#        S5       H4  n[
        R                  R                  U5      (       d    O[%        S5        M6     ['        S5      e U[         l        g! UR!                  5         Af = f! U[         l        f = f)z1Check that memmapping uses /dev/shm when possibleg    ~Ar<   rF   r   z"/dev/shm/joblib_memmapping_folder_r   r;   r  rD   r2   r  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r  r  r   r   r  r   r   rT   r  r   idr   r   r   r   r   r   )r   	orig_sizer   r   folder_prefixr_   r`   r   s           r   'test_memmapping_on_large_enough_dev_shmr0    s    11I+6 -0I)A"%	  !~~@M#..}====77>>"23333 2::.A88s?"?EE"qcBh rzz"234999
 2::.2A88s?"?EE"qcBhrzz"234999 KKMsA77>>"233#J	  !!KLL 
 -6) KKM -6)s$   G( D7G &AG( G%%G( (G5c                 z   [         R                  n [        S5      [         l        U " SSS9n UR                  nUR	                  S5      (       a   e UR                  5         A[        R                  R                  U5      (       a   e U[         l        g ! UR                  5         Af = f! U[         l        f = f)Ng@6Dr<   rF   r*  z/dev/shm)	r+  r,  r   r  r  r   r   r   r  )r   r.  r   r   s       r   $test_memmapping_on_too_small_dev_shmr2    s     11I6 -0J)A"%	  !~~'22:>>>>> KKM 77>>"233333,5) KKM
 -6)s"   B- $B 7B- B**B- -B:c                 f   [         R                  R                  nU " SSUR                  S9n UR	                  [         R
                  SS9nUR                  5       n[        U5      (       a   eU" U[         R
                  " S5      5        UR                  5         Ag! UR                  5         Af = f)z:Check that large arrays are not copied in memory in returnr<   rF   r   )  r,   r4  N)	r   r)   r*   rR   apply_asyncr   getr	   r   )r   r\   r*   r   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnr9    s     66 	bfnn=AmmBGG'm2	'....5"''$-0	 	
s   AB B0c                 .    [        U 5      (       d   eX-  $ )z4Multiplication function to be executed by subprocess)r	   )r_   n_timess     r   _worker_multiplyr<  
  s    """";r   c                    [         R                  R                  nU " SSUR                  S9n [         R                  " [         R
                  " S5      R                  S5      SS9SS2SS	2SS24   nUR                  [        US4S
9R                  5       n[        U5      (       a   eU" USU-  5        UR                  5         Ag! UR                  5         Af = f)zCheck that memmaps with a bad buffer are returned as regular arrays

Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
r<   rF   r   ip  )r4  r2   r<   r@   )rA   NrD   r5  )r   r)   r*   rR   rX   rU   rW   r6  r<  r7  r	   r   )r   r\   r*   r   r_   r`   s         r   6test_workaround_against_bad_memmap_with_copied_buffersr>    s     66bfnn=A JJryy..|<CHBQBPQR MM*!QM8<<>'****1a!e$	 	
s   BC Cc                     U $ r!   r   )args    r   identityrA  /  s    Jr   zfactory,retry_nor<   z{}, {}c                    UR                  S5      R                  nS[        R                  -  n[        R                  S-   n[	        USUSUS9nU " SUR                  S9nUR                  [        U4S	9R                  5       n[        U[        R                  5      (       d   eUR                  U:X  d   e[        R                  R                  Xh5        UR                  5         g )
Nr5   r=   rD   r6   r   )r:   r9   r8   rB   r2   r   r5  )rQ   rR   r   r   r
   r6  rA  r7  rZ   r   rS   rB   r)   r*   r   )	r   retry_nor\   r   r   rB   objr   r  s	            r    test_pool_memmap_with_big_offsetrF  3  s    & KK$,,Et)))D''!+F
e$d'&
QCv~~.A]]83&]1557Ffbii((((==F"""JJ!!#.KKMr   c                 "   Sn[        XR                  5      u  p#USL d   eX R                  S5      R                  :X  d   e[        US S9u  p#[        R                  R                  S5      (       a  USL d   eUR                  U5      (       d   eg )Ntest.tmpdirFrC  r	  )r   rR   rQ   rw   r  r  endswith)r\   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrM  S  s    $+,<nnMK++m4<<<<<+,<$OK
||u%%U""" 01111r   c                 D   SnSSK n[        UR                  R                  S5      (       a&  UR	                  UR                  R                  S5        [        USS9u  pE[        R                  R                  S5      (       a  USL d   eUR                  U5      (       d   eg)zXCheck that _get_temp_dir works when os.statvfs is not defined

Regression test for #902
rH  r   NstatvfsrC  r	  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rw   r  r  rI  )r\   monkeypatchrJ  joblibrK  rL  s         r   !test_pool_get_temp_dir_no_statvfsrV  _  s    
 %%v)),,i88F66999E+,<$OK
||u%%U""" 01111r   z1This test fails with a PermissionError on Windows	mmap_moder?   r6   c                    ^ S m[        S5       Vs/ s H  n[        R                  " SSS9PM     nn[        U SSS9" U4S	 j[	        U5       5       5      n[	        U5       H$  u  p[        R
                  R                  XA5        M&     g s  snf )
Nc                     XS S & U $ r!   r   )arrvalues     r   func4test_numpy_arrays_use_different_memory.<locals>.funcw  s    A
r   rF   )rF   rF   rT   r;   r   r2   )rW  r   r   c              3   J   >#    U  H  u  p[        T5      " X!5      v   M     g 7fr!   r   )r   r   rZ  r\  s      r   r   9test_numpy_arrays_use_different_memory.<locals>.<genexpr>}  s$      D,=&!c,=s    #)r   r   zerosr   	enumerater)   r*   )rW  r   arraysr  rZ  r\  s        @r   &test_numpy_arrays_use_different_memoryrd  q  s     <A9E9abhhxy19FEqC D,5f,=D G G$


%%c- % Fs   Bc            	      (   SS jn [         R                  " S5      n[        5       nUR                  US5        UR	                  U5      S:X  d   eUnUR	                  U5      S:X  d   eUR                  US5        UR	                  U5      S:X  d   eA[
        R                  " 5         [        UR                  5      S:X  d   eUR	                  U5      S:X  d   eAU " UR                  5        [         R                  " S5      nUR                  US5        [        UR                  5      S:X  d   eUR	                  U5      S:X  d   e[        [        5         UR	                  [         R                  " S5      5        S S S 5        AU " UR                  5        S n[        [        S	5       Vs/ s H
  oe" X&5      PM     sn5      n[        R                  " 5       S
:X  a!  [        (       a  SOSn[        U5      U:  d   eg g ! , (       d  f       N= fs  snf )Nr   c                     [        U5       H4  n[        U 5      S:X  a    g [        R                  " 5         [	        S5        M6     [        U 5      S:X  d   eg )Nr   r  )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collect>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect  sC    wA9~"JJL#J	  
 9~"""r   *   r_   r`   rD   rH   c                    [         R                  " S5      n[        [        5         U R	                  U5        S S S 5        U R                  X!5        U R	                  U5      U:X  d   e[        U5      $ ! , (       d  f       NA= f)Nrm  )r   r   r   KeyErrorr7  r   r-  )mr   r_   s      r   get_set_get_collect4test_weak_array_key_map.<locals>.get_set_get_collect  sX    GGBKHEE!H 	auuQx1}}!u	 s   A33
Br4  CPythoni  )r   )r   r   r   r   r7  rg  rh  r   _datar   ro  r   r  python_implementationr   )	rk  r_   rp  r`   rH   rq  r   
unique_idsmax_len_unique_idss	            r   test_weak_array_key_maprx    s   # 	AAEE!SM558s??	A558s??EE!SM558s??	JJLqww<1558s??	!!''*
AEE!SMqww<1558s??			bggbk 
 	
!!''* tEA)!/EFJ%%'94 %4OS:!3333 5# 
	  Fs   &G>+H>
Hc                      [        5       n [        [        R                  5         [        R                  " U 5        S S S 5        g ! , (       d  f       g = fr!   )r   r   picklePicklingErrordumps)rp  s    r   #test_weak_array_key_map_no_picklingr}    s/    A	$$	%Q 
&	%	%s   A
Ac                   ^^^ [        U R                  S5      5      m[        R                  " SSS9nUR	                  T5        U4S jmS mT" 5       n[        SS9" U4S	 jU4 5       5      n[        SS9" U4S
 jU4 5       5      n[        R                  R                  XC5        [        SS9" U4S j[        S5       5       5      n[        R                  R                  US   U5        g )Nzarr.datrF   r   r;   c                     > [        T5       n [        R                  " U R                  5       S[        R                  SS9nS S S 5        [        R
                  " S[        R                  WSS9$ ! , (       d  f       N3= f)Nr   )accessrB   r   )r8   r^   rB   )openr   filenoACCESS_READr   r[   r   )fdmmtestfiles     r   _read_array%test_direct_mmap.<locals>._read_array  sR    (^r299;$2B2B1MB zz%rxx1EE ^s   4A..
A<c                     U S-  $ Nr2   r   rK   s    r   r\  test_direct_mmap.<locals>.func      !tr   r2   r   c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr!   r_  r   rK   r\  s     r   r   #test_direct_mmap.<locals>.<genexpr>       ;s!WT]1--s   !c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr!   r_  r  s     r   r   r          A5aq!1!15r  c              3   D   >#    U  H  n[        T5      " 5       v   M     g 7fr!   r_  r   r   r  s     r   r   r          JA!5!7!7    rD   r   )	strrQ   r   rU   tofiler   r)   r*   r   r\   r_   rZ  refr  r  r\  r  s        @@@r   test_direct_mmapr    s     6;;y)*H
		"G$AHHXF
 -C !
;s;
;C a  AC5 AAGJJ!!'/ a  Jq JJGJJ!!'!*c2r   c                 ,  ^^^ [        U R                  S5      5      m[        R                  " SSS9R	                  SS5      nUR                  T5        U4S jmS mT" 5       n[        SS	9" U4S
 jU4 5       5      n[        SS	9" U4S jU4 5       5      nUS   R                  S   (       a   e[        R                  R                  XC5        [        SS	9" U4S j[        S5       5       5      n[        R                  R                  US   U5        g )Nzarr2.datrF   r   r;   r=   r2   c                  T   > [         R                  " T5      n U R                  SS5      $ )Nr=   r2   )r   rS   rW   )r  r  s    r   r  =test_parallel_memmap2d_as_memmap_1d_base.<locals>._read_array  s"    YYx zz!Qr   c                     U S-  $ r  r   r  s    r   r\  6test_parallel_memmap2d_as_memmap_1d_base.<locals>.func  r  r   r  c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr!   r_  r  s     r   r   ;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r  c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr!   r_  r  s     r   r   r    r  r  r   F_CONTIGUOUSc              3   D   >#    U  H  n[        T5      " 5       v   M     g 7fr!   r_  r  s     r   r   r    r  r  rD   )r  rQ   r   rU   rW   r  r   flagsr)   r*   r   r  s        @@@r   (test_parallel_memmap2d_as_memmap_1d_baser    s    
 6;;z*+H
		"G$,,Q2AHHX  -C !
;s;
;C a  AC5 AAGqz///JJ!!'/ a  Jq JJGJJ!!'!*a0r   )Yr   rg  	itertoolsr   r   rz  r  ru   rw   r   timer   r  rP  rR  r+  r   r   r   r   r   r	   joblib.backportsr
   joblib.executorr   TestExecutorjoblib.parallelr   r   joblib.poolr   joblib.test.commonr   r   r   r   r   joblib.testingr   r   r   r   r#   r&   r0   r3   rl   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r   r  r  r  r   parammarkr  r&  r(  r0  r2  r9  r<  r>  rA  listproductr   r   r  rt   rF  rM  rV  rd  rx  r}  r  r  )rK   ys   00r   <module>r     sm    	   	    
    (  ) F - &  7 6>/
<@ S-  S-l LLG$X*  
*Z )+  )+X C CD \99:	F#
0  0f \99:	F#
$  $N Y*F34C 5  CB Y*F34*. 5  *.Z Y*F34
F 5  
F Y*F34F 5  F !4  !4H F<  F<R Y*F34%* 5  %*P Y*F34/ 5  / Y*F342. 5  2.j \99:	F#
1  1h ++##E $ 	
 	K(  K(\ \99:	F#
  , \99:	F#
.6   .6b \99:	F#
6   60 \99:	F#
  , \99:	F#
  0 \AABE!H	
 %%'8&&A3sERSHCUV	VDA 	V	  $	22$ LLG$W [4,'. ( 
. 84 84v 3  36 1  1U	s   <P