o ^Zh.@s dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlZddlZdd lmZmZmZdd lmZdd lmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%edurdd l&m'Z'ddl(m)Z)zddl*Z*Wn e+ydZ*Ynwzddl,m-Z-Wn e+ydZ-Ynwzddl.Z.Wn e+ydZ.Ynwddl/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;mZ?e?@dddgeAe6BZCeCdde6D7ZCedurgZDnddgZDeDdgZEeFedreCGeHdddZIddZJddZKd d!ZLGd"d#d#eMZNdPd%d&ZOd'd(ZPdQd)d*ZQd+d,ZRd-d.ZSd/d0ZTd1d2ZUe"d3egd?e"d@gdAdBdCZXe"d=eCe"d>ddDgdEdFZYe"d=eCdGdHZZdIdJZ[ee"dKgdLdMdNZ\ee"d=gdOdPdQZ]dRdSZ^e"dTe6e"dUe6dVdWZ_dXdYZ`edZd[Zad\d]Zbe"d>gd^d_d`Zce"d=eEdadbZdedcddZeeee"degdfe"dgddhgdidjZfe"d=eEdkdlZgee"d=eEdmdnZhee"d=eie?dohe"dpdqdrgdsdtZjee"d=eDdudvZkee"d=e6dwdxZldydzZme"d=e6e"d{dgd|fd}gd~fgddZnee"d=eEddZoddZpee"d=eDddZqddZrddZsddZtee"d=eEejVjuddddZvGddde3ZwddZxe"d=eCddZyee"d=eEe"d>gdddZzddZ{ddZ|e$eduddddZ}ddZ~eEddZeddedDee"d=ee"d3e;egde"d=eEe"d3egdddZddZddZejV"d>dDd}gee"d=eie?dohddZe"d=eCe"d>gdddZddZddZee"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"ddqdrge"ddqdrge"d>gdd d Ze"d gd e"ddqdrge"ddqdrgd dZeee"d=eDddZe"d=edurBddgngdddZee"d=eDddZee"d3egd6d7d8Zeee"d9gd:e"d>dDd6ge"d3edDd6ge"d=gd>e"d3egdAdBdCZːdDdEZ̐dPdFdGZee"d>dDd}ge"d=eDe"d3edDd}ge"d=eDdJdKZeejV"d>dDd}gdLdMZeejV"d>dDd}gdNdOZdS(Tz Test the parallel module. N) nullcontextsqrt TimeoutError) PicklingError)sleep)format_exception)dumploadparallel)mp)IS_GIL_DISABLEDnpwith_multiprocessing with_numpy)check_subprocess_call parametrizeraisesskipifwarns)get_reusable_executor)Queue) parallel_sum) LokyBackendMultiprocessingBackendParallelBackendBaseSequentialBackendThreadingBackend) BACKENDSParallel cpu_countdelayedeffective_n_jobsr parallel_backendparallel_configregister_parallel_backendmultiprocessingcCsg|]}t|qS)r ).0Z backend_strr)r)H/var/www/auris/lib/python3.10/site-packages/joblib/test/test_parallel.py Rsr,loky threading get_contextspawncCs ttjSN)r r DEFAULT_BACKENDr)r)r)r+get_default_backend_instance^s r3cCst|dt|ddS)N_poolZ_workers)getattrbackendr)r)r+ get_workersfsr8cCs||Sr1r))xyr)r)r+divisionjr;cC|dSNr)r9r)r)r+squarenr<rAc@seZdZdZddZdS)MyExceptionWithFinickyInitz,An exception class with non trivial __init__cCdSr1r))selfabcdr)r)r+__init__uz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rIr)r)r)r+rBr rBFcCs"|dkr|r tddddt|S)NrErFrGrH)rB ValueError)r9custom_exceptionr)r)r+exception_raiserysrTcCstdt)Ng?)timerKeyboardInterruptr@r)r)r+interrupt_raisers rWcCs|d||S)zNA module-level function so that it can be spawn with multiprocessing. r?r))r9r:zr)r)r+fsrYcCsttdSNr)typer get_active_backendr)r)r)r+_active_backend_typer]cCst||dddtdDS)Nn_jobsr7cs|] }tt|VqdSr1r#rAr*ir)r)r+   z parallel_func..r!range) inner_n_jobsr7r)r)r+ parallel_funcsrkcCtdksJdSrZ)r"r)r)r)r+test_cpu_countrmcCrlrZr$r)r)r)r+test_effective_n_jobsrnrpcontextzbackend_n_jobs, expected_n_jobs)rgrgr`Nr)z positive-intz negative-intNone)ZidscCsT|d|dtdd|ksJWdn1swYtdddks(JdS)Nr.rsrro)rqZbackend_n_jobsZexpected_n_jobsr)r)r+test_effective_n_jobs_Nones rvr7r`)rr?rrverbose)r? dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr)rAr*r9r)r)r+r,z(test_simple_parallel..r`r7rxcsrar1rbr}r)r)r+rez'test_simple_parallel..rir!)r7r`rxr)r)r+test_simple_parallels rr?c Csd}td}Gdddt}|||dd}|ddt|Dt}|jD]}||}r=|\} } || | q't |d ksFJdS) Nrzz(Done\s+\d+ out of \d+ \|)c@seZdZgZddZdS)z/test_parallel_pretty_print..ParallelLogcSs|j|dSr1)messagesappend)rDmsgr)r)r+_printr^z6test_parallel_pretty_print..ParallelLog._printN)rLrMrNrrr)r)r)r+ ParallelLogrPri'rcSg|]}tt|qSr)r#rYrcr)r)r+r,z.test_parallel_pretty_print..r) recompiler!risetrsearchspanaddlen) r7r`n_taskspatternrexecutorZlensmessagesrErFr)r)r+test_parallel_pretty_prints   rcs|jtdddtjdd}td|ddd td D}|gd ks(JWdn1s2wYd d |D}|dvsGt|trRdfdd |D}t |dksZJdS)NnameZ!some_new_name_for_the_main_thread)targetrvalueTrecordr?r_csrar1rbr}r)r)r+rerfz6test_main_thread_renamed_no_warning..rg)rrcSs*g|]}dt|jvrt|jts|qS)zworker timeout)strr isinstanceDeprecationWarningr*wr)r)r+r,s z7test_main_thread_renamed_no_warning..)Nr(z3multi-threaded, use of fork() may lead to deadlockscsg|] }t|jvr|qSr))rrr message_partr)r+r,r) setattrr.current_threadwarningscatch_warningsr!rirrr)r7 monkeypatchwarninforesultsr)rr+#test_main_thread_renamed_no_warnings(  rcCstjdd}tdt||dWdn1swYdd|D}|rI|rGtdd|D}tr=t|d knt|d k}|oF|Sd S|rMJdS) NTralways)r7rjcSsg|]}|jqSr))rrr)r)r+r,sz*_assert_warning_nested..css|] }d|jdvVqdS)zbacked parallel loops cannotrN)args)r*Zeachr)r)r+re  z)_assert_warning_nested..rF)rr simplefilterrkallrr)r7rjexpectedrZwarnings_are_correctZwarnings_have_the_right_lengthr)r)r+_assert_warning_nesteds  rz%parent_backend,child_backend,expected))r-r(T)r-r-F)r(r(T)r(r-T)r.r(T)r.r-Tcsntd|dfddtdDtd|dfddtdD}|dkr/t|s-JdSt|s5JdS)Nr?r_c3s"|] }ttdddVqdS)rFr7rjrNr#rr*_ child_backendr)r+re( z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)r?rNrrrrr)r+re0rr.)r!rianyr)parent_backendrrresr)rr+test_nested_parallel_warningss r)r-r(r.cCs>dg}dd}tj||fd}|||dsJdS)NFcSshtjdd}tddddtdDWdn1swYtt|t|dk|d<dS) NTrr?rscs|] }ttdVqdS)皙?Nr#rrr)r)r+reFrzPtest_background_thread_parallelism..background_thread..rr)rrr!riprintr)is_run_parallelrr)r)r+background_threadDs  z=test_background_thread_parallelism..background_thread)rrr)r.Threadstartjoin)r7rrtr)r)r+"test_background_thread_parallelism?s rcCs"td|dddtdDdS)Nr?r_csr{Gz?Nrbrr)r)r+reQrznested_loop..rhr6r)r)r+ nested_loopPs"rrrcs&td|dfddtdDdS)Nr?r_c3|] }ttVqdSr1)r#rrrr)r+reWrfz#test_nested_loop..rh)rrr)rr+test_nested_loopTs rcCstr1)rRr6r)r)r+raise_exception\rJrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr!rifullr)r)rr+test_mutate_input_with_threadsgs "r)rr?rgcCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrr:rYr}r)r)r+r,rrz(test_parallel_kwargs..rscs |] }tt|ddVqdSrrNrr}r)r)r+rer z'test_parallel_kwargs..Nr)r`lstr)r)r+test_parallel_kwargsnsrcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrr}r)r)r+r,zrz4test_parallel_as_context_manager..rr_csrrrr}r)r)r+rez3test_parallel_as_context_manager..csrrrr}r)r)r+rercsrrrr}r)r)r+rer)rir!_backendr r8)r7rrpZmanaged_backendr)r)r+ test_parallel_as_context_managerws"rcsdGdddtttddtdddfdd td DWd d S1s+wYd S) z`Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)NZ123) RuntimeErrorrDr)r)r+ __reduce__r<z.UnpicklableObject.__reduce__N)rLrMrNrr)r)r)r+UnpicklableObjects rzthe task to sendmatchr?r-r_c3s|] }ttVqdSr1r#idrrr)r+rerz)test_parallel_pickling..rN)objectrrr!rir)r)rr+test_parallel_picklings  "r byteorder)<>= max_nbytesZ1Mcsddtdd|djj}tdd|dfd d td D}|D]\}}||ks5J||jjks=Jtj |q+dS) NcSs ||jjfSr1)dtyperr@r)r)r+inspect_byteorder z=test_parallel_byteorder_corruption..inspect_byteorder)r?rgi4r?r-)r`r7rc3|] }tVqdSr1r#rrr9r)r+rerfz5test_parallel_byteorder_corruption..rg) rZarangeZreshapeviewrrr!ritestingassert_array_equal)rrZinitial_np_byteorderresultZ x_returnedZbyteorder_in_workerr)rr+"test_parallel_byteorder_corruptions  rcCs0ttd|ddddtdDdksJdS)Nr?r`r7timeoutcsr)gMbP?Nrr}r)r)r+rerfz0test_parallel_timeout_success..r)rr!rir6r)r)r+test_parallel_timeout_successs r cCsNtttd|ddddtdDWddS1s wYdS)Nr?rr csrrNrr}r)r)r+rerfz-test_parallel_timeout_fail..r)rrr!rir6r)r)r+test_parallel_timeout_fails  "r  sequential return_as generatorgenerator_unorderedcCsxttttd||ddddtdDWdn1s"wYttd||ddddtdDdS)Nr?r)r`r7rr csrr rr}r)r)r+rerfz.rcsrrrr}r)r)r+rerf)rrlistr!rir7rr)r)r+)test_parallel_timeout_fail_with_generators rc Cstdurtttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJ|j |j |j |j fWdn 1swYt|jdusJn!tttdddddDWdn 1s2wYtttddddtddDWdn 1sVwYtt tddddd tdDWddS1s{wYdS)Nr?r_cSg|] \}}tt||qSr)r#r;r*r9r:r)r)r+r,rz&test_error_capture..)rr)rrcSrr)r#rWr}r)r)r+r,rcSrr)rrr)r)r+r,rcSrrrr}r)r)r+r,rrcsrrrr}r)r)r+rerz%test_error_capture..cSrr)rr}r)r)r+r, rcSrrrr}r)r)r+r,rcsrrrr}r)r)r+rerrscSrr)rr}r)r)r+r,"rcSrr)rrr)r)r+r,'rr)r`rxcsr)T)rSNr#rTrcr)r)r+re+rr)r rZeroDivisionErrorr!ziprVr8rri _iteratingZn_completed_tasksZn_dispatched_tasks _abortingrB)r7r Zoriginal_workersr)r)r+test_error_capturesd           '     $rc Csddd}td|dw}ttdd|dd |dd DWdn1s)wYttdd|d d |d d DWdn1sKwYttdd|d d |dd DWdn1smwYWddSWddS1swYdS)Nrcss*tdD] }||krtd|VqdS)NIterator Raising Error)rirR)raise_atrdr)r)r+ my_generator2s  z1test_error_in_task_iterator..my_generatorr?r_r rcsrar1rbrcr)r)r+re;rz.test_error_in_task_iterator..)r!csrar1rbrcr)r)r+re@rrcsrar1rbrcr)r)r+reDrr)r!rrR)r7r"rr)r)r+test_error_in_task_iterator/s  "r%cCs|d|dS)Nz Consumed %s)r)queueitemr)r)r+consumerGrnr(zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) r)r+r-r/r*r,r.r0r1r3r2r4csTtfdd}td||dfdd|D|ks Jtdks(JdS) z=Test that with only one job, Parallel does act as a iterator.c3(tdD] }d||VqdSNrz Produced %irirrdr&r)r+producerx  z'test_dispatch_one_job..producerrr` batch_sizer7c3s|] }tt|VqdSr1r#r(r}r9r)r+re}rz(test_dispatch_one_job.. N)rr!r)r7r=Zexpected_queuer:r)r9r+test_dispatch_one_jobKs+  r@cst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zRCheck that using pre_dispatch Parallel does indeed dispatch items lazily. c3r5r6r7r8r9r)r+r:r;z/test_dispatch_multiprocessing..producerr?rrg)r`r= pre_dispatchr7c3s|] }ttdVqdS)rNr>rr9r)r+rerz0test_dispatch_multiprocessing..rr)Nrz Consumed anyrrr/r?)r Managerrr!indexr)r7managerr:Zqueue_contentsZfirst_consumption_indexZproduced_3_indexr)r9r+test_dispatch_multiprocessings    rEcCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) Nr?autor.r<csrar1rrcr)r)r+rerz/test_batching_auto_threading..rr!rirZcompute_batch_size)rr)r)r+test_batching_auto_threadings"rIcCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)Nr?rFr<csrar1rrcr)r)r+rerz2test_batching_auto_subprocesses..rGrrH)r7rr)r)r+test_batching_auto_subprocessess"rJcCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedr?r)r`rArxcsrar1rrcr)r)r+rerfz*test_exception_dispatch..rN)rrRr!rir)r)r)r+test_exception_dispatchs  "rLcC tddddtdDdS)Nr?rscsrar1rr*jr)r)r+rerz(nested_function_inner..rrhr8r)r)r+nested_function_inner rPcCrM)Nr?rscsrar1)r#rPrNr)r)r+rerz(nested_function_outer..rrhr8r)r)r+nested_function_outerrQrRz'https://github.com/joblib/loky/pull/255reasoncCstt}td|dddtdDWdn1swYt|j|j|j}d|}d|vs7Jd |vs=Jd |vsCJt|jtusLJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r?r_csrar1)r#rRrcr)r)r+rerfz1test_nested_exception_dispatch..rNrRrPrT) rrRr!rir r[rtbr)r7excinfoZ report_linesreportr)r)r+test_nested_exception_dispatchs      rYc@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr1)r$r`r )rDr`r Z backend_argsr)r)r+ configures zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr r"rDr`r)r)r+r$sz$FakeParallelBackend.effective_n_jobsr)r)rLrMrNrOr[r$r)r)r)r+rZs rZc Csttdd tddWdn1swYttddtddWdn1s2wYWdn1sAwYttdd&tddWdn1s]wYWddSWddS1suwYdS)NzInvalid backend:rz unit-testingr6)rrRr!r&r)r)r)r+test_invalid_backends    "r^cCs,tt}td|dWdn1swYdt|jvs%Jtt}td|dWdn1s.rcss |] \}}|t|kVqdSr1r|)r*rdrr)r)r+rer)r!_effective_n_jobsrir enumerate)r7r`rrr)r)r+test_njobs_converted_to_ints recCs<ztdtdtvs JtdtksJWtd=dStd=w)NZ test_backend)r'rZr r)r)r)r+test_register_parallel_backends   rfcCs^tj}ttks JztdtdddttksJW|t_n|t_wttks-JdS)Nr.T)Z make_default)r r2r]r3r'r r)Zdefault_backend_origr)r)r+test_overwrite_default_backend%srgzOnly without multiprocessingcCsttddtddddtdDWdn1swYtddtddtdDWddS1sAwYdS) Nz)joblib backend '.*' is not available on.*rr-r6csrar1rbrcr)r)r+re4rz2test_backend_no_multiprocessing..rgcsrar1rbrcr)r)r+re8r)r UserWarningr!rir&r)r)r)r+test_backend_no_multiprocessing1s  "ricCs^||ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tus1Jt|jtus:JnB|dkrQt|tusGJt|jtusPJn4|dkrgt|t us]Jt|jt usfJn.| drt|t ustJt|jt usJWddSWddSWddSWddSWddS1swYdS)Nrgrsr(r-r.Ztest_) r r\r$r!r`r[rrrr startswithrZ)rq backend_nameactive_backend active_n_jobsrr)r)r+check_backend_context_manager;s8    "rncCsg|]}d|qS)ztest_backend_%dr)rcr)r)r+r,Qr~rgcsr|tvr |t|tttksJt|ttks JtdddfddtDttks7JdS)Nr?r.r_c3s"|] }|stt|VqdSr1)r#rn)r*rFrqr)r+recs  z/test_backend_context_manager..)r setitemrZr]r3rnr! all_backends_for_context_manager)rr7rqr)ror+test_backend_context_managerTs rrc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rRparam)rDrtr)r)r+rIps z%ParameterizedParallelBackend.__init__r1rKr)r)r)r+rsmsrscCs|tdtttksJ|dddd?t\}}t|tus%J|jdks,J|dks2Jt }|j dks.rcSr{r)rrcr)r)r+r,r~z>test_parameterized_backend_context_manager..) rpr rsr]r3r r\r[rtr!r`rri)rrqrlrmrrr)r)r+*test_parameterized_backend_context_managervs   rwcCsttksJ|tdddd?t\}}t|tus J|jdks'J|dks-Jt}|jdks7J|j |us>J|ddt dD}Wdn1sSwY|ddt dDkseJttksmJdS) N+)rtrrscsrar1rvrcr)r)r+rerzFtest_directly_parameterized_backend_context_manager..cSr{r)rrcr)r)r+r,r~zGtest_directly_parameterized_backend_context_manager..) r]r3rsr r\r[rtr!r`rri)rqrlrmrrr)r)r+3test_directly_parameterized_backend_context_managers   rycCstdtS)Nr)rosgetpidr)r)r)r+sleep_and_return_pidsr|cCs<ttksJtdksJtddddtdDS)Nrr?rscs|]}ttVqdSr1)r#r|rr)r)r+rez"get_nested_pids..)r]rr!rcrir)r)r)r+get_nested_pidssrcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrZ)superrget_nested_backendr __class__r)r+rszMyBackend.get_nested_backend)rLrMrNrOr __classcell__r)r)rr+rsrback_compat_backend)r.r-r(rcCsh||&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)Nr?rscsr}r1)r#rrr)r)r+rer~z6test_nested_backend_context_manager..rr)r!rirr)rqr7Z pid_groupsZ pid_groupr)r)r+#test_nested_backend_context_managers "r)r?rrNcs~ddtddfddtdD|dtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSr1)r]r r$r!rc)Zexpected_backend_typeZexpected_n_jobr)r)r+check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrrsc3s |] }ttjdVqdSr)r#r r2r)rr)r+rerz4test_nested_backend_in_sequential..rc3s|] }tVqdSr1rrr7rr`r)r+rerrh)r7r`rqr)rr+!test_nested_backend_in_sequentials  "rcCs\|| }|tur |d}|tur|d}|j|ksJWddS1s'wYdS)Nr7r)r&r% nesting_level)rq inner_backendZexpected_levelctxr7r)r)r+check_nesting_levels "r outer_backendrcst|dtd|dfddtdDddtfddtdDWddS1s9wYdS) Nrr?r_c3 |] }ttdVqdSrr#rrrqrr)r+rerz-test_backend_nesting_level..rrsc3rrrrrr)r+rer)rr!ri)rqrrr)rr+test_backend_nesting_levels  "rwith_retrieve_callbackTcsddlGfdddt}td|dd|d;}tddfd d td D|tur:|d jd ks:J|turG|djd ksOJWddSWddS1sZwYdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrtr8rr)r)r+retrieval_context s z;test_retrieval_context..MyBackend.retrieval_contextN)rLrMrNrdZsupports_retrieve_callbackcontextmanagerrr)) contextlibrr)r+rs rZ retrievalcSstddddt|DS)Nr?rscsrar1rrcr)r)r+rerz>test_retrieval_context..nested_call..rh)nr)r)r+ nested_callsz+test_retrieval_context..nested_callr?rsc3s|] }t|VqdSr1rrc)rr)r+rerz)test_retrieval_context..rr7r)rrr'r!rir&rdr%)rqrrrr))rrrr+test_retrieval_contexts   "rr=)rrrgQ?cCs8ttt|dWddS1swYdS)Nr=)rrRr!rr)r)r+test_invalid_batch_sizes  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )r?r?rrF)r?r?r`rFrr?r`rF)ir?r`rFr)rrr`rF)r?r`rF)r? 2 * n_jobsr)r?rr)rr?rrQ)rr?rrFcCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)r`rAr=cSr{r)r|rcr)r)r+r,:r~z0test_dispatch_race_condition..csrar1rbrcr)r)r+re;rz/test_dispatch_race_condition..r)r)rr`rAr=paramsrrr)r)r+test_dispatch_race_condition%s  rcCs8t}tddd}|jd}|}||ksJdS)Nr?r(r_rq)r get_start_methodr!Z_backend_kwargsget)Zmp_start_methodrrqZ start_methodr)r)r+test_default_mp_context?s   rcs^|dkr td}tjd}|ddtjtd|dfddt dDdS) Nr(r0ruir?r_c3s"|] }ttjjVqdSr1)r#rdotTrcrEr)r+re^s zAtest_no_blas_crash_or_freeze_with_subprocesses..) r r/rrandomZ RandomStaterandnrrr!ri)r7rngr)rr+.test_no_blas_crash_or_freeze_with_subprocessesHs    &rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr(forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r  stdout_regex) r rpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r7coder)r)r+2test_parallel_with_interactively_defined_functionsqs  raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r(r0win32zNot picklable with pickle)rr7rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrzpathdirnamejoblib__file__rwriterrstrpath)r7rrtmpdirr code_filer)r)r+0test_parallel_with_unpicklable_functions_in_argss      raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr )rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr)r)r+7test_parallel_with_interactively_defined_functions_lokys    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)rZ stderr_regexr )rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr)r)r+:test_parallel_with_interactively_defined_bound_method_lokys    rcCs"tg}tdd|gksJdS)Nr?rs)iterr!)Zexhausted_iteratorr)r)r+%test_parallel_with_exhausted_iterator)srcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorr[copyrr)r)r+ check_memmap.s rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrr)rironesZfloat32)rrdr)r)r+generate_arrays<s zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr?rr`rr7csrar1r#rr*rEr)r)r+reBrfz.rzrcsrar1rrr)r)r+reKrf)r!rrrrr)r7rrrrr)r)r+)test_auto_memmap_on_arrays_from_generator4srcCs|Sr1r))argr)r)r+identityRrJrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapZuint8rrb)Z mmap_moder?rsc3rr1)r#rrrr)r+re^rz.test_memmap_with_big_offset..r$r)rrmmapALLOCATIONGRANULARITYrZzerosrr r r!rroffsetrr)rfnamesizeobjrr)rr+test_memmap_with_big_offsetVs   rcCstjdd}tdddddtdDWdn1s wYt|dks-J|d}t|jts9Jt|jd ksBJdS) NTrr)r`r csrar1rbrcr)r)r+refrzFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr!rirrrrhr)rrr)r)r+3test_warning_about_timeout_not_supported_by_backendds  rcCs |||<|Sr1r)) input_listrCrr)r)r+set_list_valueqsr)rr?rcsVdgdt|dddfddtdD}t|}tddt|Ds)JdS) Nrrrr.r`rr7c3s |] }tt||VqdSr1)r#rrcrr)r+re|rzPtest_parallel_return_order_with_return_as_generator_parameter..cs|] \}}||kVqdSr1r)r*vrbr)r)r+rer)r!rirrr)r`rr)rr+=test_parallel_return_order_with_return_as_generator_parametervs  rcCs|rtdt|S)Nr)rr)edelayr)r)r+_sqrt_with_delaysrcspt|d|dddtdDtfddtdD}dgttd d}td dt||Ds5JdS) Nrrcss&|]}tt|d|dkVqdS)r?rN)r#rrcr)r)r+res zK_test_parallel_unordered_generator_returns_fastest_first..rc3s|]}tVqdSr1)nextrrr)r+res rr?csrr1r)rr)r)r+rer)r!risortedrrr)r7r`Zquickly_returnedZexpected_quickly_returnedr)rr+8_test_parallel_unordered_generator_returns_fastest_firstsrcCst||dSr1)rr7r`r)r)r+7test_parallel_unordered_generator_returns_fastest_firstsr)rr?rwrrcCsvdgdgd}ttt}t||ddd|DWdn1s(wYt|}|dks9JdS)NrErrzr_css|] }ttj|VqdSr1r#rUrrcr)r)r+rerz%test_abort_backend..r)rrrUr!)r`r7Zdelayst_startdtr)r)r+test_abort_backends  rcCstjtdtd}d|d<|S)NgArFr)rrintbool)rrr)r)r+get_large_objectsrcCs\t|||d}|ddtdD}t|t|~WddS1s'wYdS)N)r`r7rcsrar1)r#rrcr)r)r+rerz0_test_deadlock_with_generator..r)r!rir)r7rr`r rr)r)r+_test_deadlock_with_generators "rcCst|||dSr1)r)r7rr`r)r)r+test_deadlock_with_generatorsrcCsttdd)t|||d}|ddtdD}t}|ddtdD}Wdn1s2wYt|d ksCJd ~dS) N)This Parallel instance is already runningrrcsrrrrr)r)r+rerz/test_multiple_generator_call..rcsrar1rrcr)r)r+rerrzr?[The error should be raised immediately when submitting a new task but it took more than 2s.)rrr!rirU)r7rr`r grZgen2r)r)r+test_multiple_generator_callsr c Cst|||dF}|ddtdD}t}ttdd|ddtdD}Wdn1s3wYt|d ksDJd Wd~dS1sPwY~dS) Nrcsrr rrr)r)r+rerz7test_multiple_generator_call_managed..rrrcsrar1rrcr)r)r+rerrzr?r)r!rirUrr)r7rr`r rrg2r)r)r+$test_multiple_generator_call_manageds   r  return_as_1 return_as_2cCst|||dddtdD}t|||dddtddD}|dkr)t|}|dkr1t|}tddt|tdDsAJtd dt|tddDsRJdS) Nrcs |] }tt|dVqdSr?Nrvrcr)r)r+rerz9test_multiple_generator_call_separated..rcsrrrvrcr)r)r+rerrrcsrr1r)r*rrdr)r)r+rercsrr1r)rr)r)r+rer)r!rirrr)r7r r r`rr r)r)r+&test_multiple_generator_call_separateds &rzbackend, error))r-T)r.F)rFc Cs\|dkr tdur tdtd||d}|ddtdD}t|dd }|r/ttd d nt }|6t }td||dd dtdd D}|dkrRt |}t ddt |tdd DscJWdn1smwYt |dks|Jd} |jr| dkr| d7} t d|jr| dkst |dksJ|dkr|jsJdSdS)Nr-zRequires multiprocessingr?rcsrr rrcr)r)r+re&rz.rcSstdS)NzGenerator collected)rr)r)r)r+'z;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcsrrrvrcr)r)r+re3rrrcsrr1r)rr)r)r+re:rrrrgrr_)r rrr!riweakreffinalizerrrrUrrraliverrcr) r7r r errorr rZg_wrrrretryr)r)r+)test_multiple_generator_call_separated_gcs: $   rcCs|j}tdd||d#}|ddtjdgdDtt|dks'JWdn1s1wYtdD] }t|sCn td q:t d tdd|d }|d dtjdgdDtdD]}t|sqdStd qgt d ) Nr?r)r`rr7Z temp_foldercsrar1rrr)r)r+reVrz(test_memmapping_leaks..rrrzrz/temporary directory of Parallel was not removedrcsrar1rrr)r)r+refr) rr!rrrrzlistdirrirAssertionError)r7rrrr)r)r+test_memmapping_leaksKs"        r)Nr-r.cCs<td|dddtdD}|ddtdDksJdS)Nr?r_css |] }tdd|VqdS)cSr=r>r)r@r)r)r+rvrz2test_lambda_expression...Nrrcr)r)r+reurz)test_lambda_expression..rcSsg|]}|dqS)r?r)rcr)r)r+r,xr~z*test_lambda_expression..rh)r7rr)r)r+test_lambda_expressionpsrcsd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.r?ig@r)rxr`r7c3|] }ttjVqdSr1rrc task_timer)r+rerz6test_backend_batch_statistics_reset..c3rr1rrcrr)r+rerN)r!rirZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r7r`Zn_inputsrr)rr+#test_backend_batch_statistics_reset{s r!cCsdD]<}tt|djtusJt|dd}t|jtus Jt|dd}t|jtus/Jt|dd}t|jtus>Jqtdd dd }t|jtusOJ|d ddtdd }t|jtusdJ|jdkskJWdn1suwY|d ddtd dd}t|jtusJ|jd ksJWdn1swY|d ddtdd }t|jtusJ|jdksJWdn1swY|d ddtd dd}t|jtusJ|jd ksJWddS1swYdS)N)rr?rrrsthreadsr`prefer processes sharedmemr`requirer?r-)r`r7r$r$rgr(r)r[r!rr3rrr`)rqr`rr)r)r+$test_backend_hinting_and_constraintss<       "r+cCszGdddt}||&tddd}t|j|usJtddd}t|j|us,JWdn1s6wYGdd d t}||Otddd}t|j|usXJ|\}}|d ksdJ|d ksjJtddd d }t|jtuszJ|\}}d }||ksJ|d ksJWdn1swYttt|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrCr1r)rr)r)r+ apply_asyncrJzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|Sr1r)r]r)r)r+r$rJzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrLrMrNZsupports_sharedmemZ use_threadsr-r$r)r)r)r+MyCustomThreadingBackend  r0r?r%r#r&r'c@r,)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrCr1r)rr)r)r+r-rJzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSr.r1r)r]r)r)r+r$rJzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNr/r)r)r)r+MyCustomProcessingBackendr1r2rUr)r`r(rxzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r7r() rr!r[rZ readouterrrstriprrR)Zcapsysrqr0rr2outerrrr)r)r+9test_backend_hinting_and_constraints_with_custom_backendss2        "r7cCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtdurtttdddWdn1sewYtttd ddWddS1swYdSdS) Ninvalidr)r*r%r&)r$r(r-r3r()rrRr!r r)r)r)r+,test_invalid_backend_hinting_and_constraintss"       "r9c stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r!r[rrLrri)r:rrZ this_levelrr)r<r+r;s  $r;cCs^||dd t}Wdn1swY|d}|dfdddg}||ks-JdS)Nr?rsBackendr)rr)rr?)r;title)rqr7Zbackend_types_and_levelsZtop_level_backend_typeZexpected_types_and_levelsr)r)r+test_nested_parallelism_limits r?cCstddtdDS)z6A horrible function that does recursive parallel callscsr}r1)r#_recursive_parallelrcr)r)r+re0r~z&_recursive_parallel..r?rh)Z nesting_limitr)r)r+r@.sr@c Cs||ddtt }tWdn1swYWdn1s(wY|j}|dkrFddlm}t||tfrFt dt|t sMJdS)Nr?rsr-r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionr@rZ&joblib.externals.loky.process_executorrArrrxfailRecursionError)rqr7rWexcrAr)r)r+test_thread_bomb_mitigation3s   rFcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrz)rzenvironrr)Zenv_varsvarr)r)r+_run_parallel_sumOs rMzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) Nr?r_csr}r1)r#rMrr)r)r+rea  z-test_parallel_thread_limit..rZ_THREADSrJ1)r!rir\r"itemsendswithr)r7rZexpected_num_threadsZworker_env_varsZomp_num_threadsrrr)r)r+test_parallel_thread_limit^s    rRzThis test requires daskcCs:ttdd |dWddS1swYdS)NzPlease install daskrZdask)rrRror)r)r+)test_dask_backend_when_dask_not_installedos "rSc CsGdddt}d}||6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_dSrZr))rDrrr)r)r+r[{rJz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rDfunccallbackr)r)r+r-~r<z?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrTrZr)r]r)r)r+r$rJzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsr1)rLrMrNr[r-r$r)r)r)r+ZeroWorkerBackendzs  rWz&ZeroWorkerBackend has no active workerrr?rscsrar1rrcr)r)r+rerz+test_zero_worker_backend..)rrrrr!ri)rqrWZ expected_msgr)r)r+test_zero_worker_backendvs "rXcsdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSr1)MY_GLOBAL_VARIABLEr)r)r)r+ check_globalssz@test_globals_update_at_each_parallel_call..check_globalsr?rsc3|]}tVqdSr1rrcrZr)r+rerNz.z changed valuec3r[r1rrcr\r)r+rerN)rYr!rir)Zworkers_global_variabler)r\r+)test_globals_update_at_each_parallel_callsr]cCs4ddl}|jdd}|||td}|S)Nrrz threadpoolctl)numpyrrrrZ importorskipZthreadpool_info)rrEr^r)r)r+_check_numpy_threadpool_limitss   r`cCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rRr) child_module parent_info parent_moduler)r)r+_parent_max_num_threads_fors rfcCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nrb)rfmin)Z workers_infordrbZchild_threadpool_inforcZparent_max_num_threadsrr)r)r+check_child_num_threadssrh)r?rrwrrcCszt}t|dkrtjddtd|dddtdD}t|}|d kr-|dd }ntt|d }t |||dS) Nr&Need a version of numpy linked to BLASrSr-rcsr}r1r#r`rcr)r)r+rerNz;test_threadpool_limitation_in_child_loky..r?rrb r`rrrr!rir$r\r"rh)r`rdworkers_threadpool_infosexpected_child_num_threadsr)r)r+(test_threadpool_limitation_in_child_lokys  rninner_max_num_threads)rr?rNcCst}t|dkrtjdd|d|dt|dddtd D}Wdn1s.wYt|}|d krB|dd }n|durOtt|d }n|}t |||dS) NrrirSr-rorscsr}r1rjrcr)r)r+rerNz>test_threadpool_limitation_in_child_context..r?rrbrk)rqr`rordrlrmr)r)r++test_threadpool_limitation_in_child_contexts"   rqvar_name)rHrIrGc st|dkr tdtddddtj}z_dtj<t|dfd d t d D}|ddgks;J|d dd t|dfdd t d D}Wdn1s]wY|ddgksjJW|durutj=dS|tj<dS|durtj=w|tj<w)NrzSkip test when n_jobs == 1TZreusecSs tj|Sr1)rzrKr)rrr)r)r+_get_envrz>test_threadpool_limitation_in_child_override.._get_env4rsc3rr1rrcrtrrr)r+re"rz?test_threadpool_limitation_in_child_override..r?r-rpc3rr1rrcrvr)r+re&rfrO) r$rrrshutdownrzrKrr!ri)rqr`rrZoriginal_var_valuerr)rvr+,test_threadpool_limitation_in_child_override s*   "  rx)r?rrrcCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrrsc3rr1)r#sumrcr@r)r+re:rzAtest_loky_reuse_workers..parallel_call..rrsr)r@r+ parallel_call8s$z.test_loky_reuse_workers..parallel_callTrsr)rri)r`rzZfirst_executorrrr)r)r+test_loky_reuse_workers2s   r{cCsd|t<dS)N initialized)rzr{)statusr)r)r+_set_initializedHr^r~cCst}||d}|dvsJd||sdSd||<td}tdd|D}t|krR||krRtdtdd|D}t|krR||ks8t|kr\td |S) N)r|startedz2worker should have been in initialized state, got rrcSg|] \}}|dkr|qSrr)r*pidrr)r)r+r,Xz!_check_status..rcSrrr)rr)r)r+r,[rz-Waited more than 30s to start all the workers)rzr{rrUrrPrr)r}r` wait_workersrstatedeadlineZ n_startedr)r)r+ _check_statusLs"     rcsft}|||tfdtfddtdDWddS1s,wYdS)Nr7r`Z initializerZinitargsc3|] }ttVqdSr1r#rrcr`r}r)r+resrz+test_initializer_context..rz)r rBdictr~r!ri)r`r7rqrDr)rr+test_initializer_contextcs "rcs>t}|t|tfdfddtdDdS)Nrc3rr1rrcrr)r+rerz,test_initializer_parallel..rz)r rBrr!r~ri)r`r7rDr)rr+test_initializer_parallelvsrcsxd}t}|t}t|D]}tdtfdfddtD}|t|}qt|ks:JddS)Nrgr-rc3"|] }ttddVqdST)rNrrcrr)r+re  z*test_initializer_reused..z=The workers should be reused when the initializer is the same) r rBrrrir!r~unionrr`Z n_repetitionsrDZpidsrdrr)rr+test_initializer_reuseds&   rcs|d}t}t}t|D]"}|tdtfdfddtD}|t|}q t||ks.zFThe workers should not be reused when the initializer arguments change) r rBrrirr!r~rrrr)rr+test_initializer_not_reuseds&   r)F)rr)rgr1)rOrrzrrr.rUrrrrmathrr(rpicklerr tracebackr rrr r r Zjoblib._multiprocessing_helpersr Zjoblib.test.commonrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrr&rposix ImportErrorZ _openmp_test_helper.parallel_sumr distributedZjoblib._parallel_backendsrrrrrZjoblib.parallelr r!r"r#r$r%r&r'rZRETURN_GENERATOR_BACKENDSpoprkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrrr/r3r8r;rA ExceptionrBrTrWrYr]rkrmrpmarkrvrrrrrrrrrrrrrrrr r rrrr%r(r@rErIrJrLrPrRrCrYrZr^r`rerfrgrirnrqextendrirrrsrwryr|rZ_parallel_backendsrrrrrrrrrrrrrrZ SQUARE_LOCALZ SQUARE_LAMBDArrrrrrrrrrrrrrrrrrrrrrrrr r rrrrr!r+r7r9r;r?r@Zno_coverrFrMrRrSrXr]r`rfrhrnrqrxr{r~rrrrrr)r)r)r+s              ,            *                F  (                              (+                    ,"  . 6        $      #