Your IP : 18.191.171.121

Current Path : /lib64/python3.8/multiprocessing/__pycache__/
Upload File :
Current File : //lib64/python3.8/multiprocessing/__pycache__/pool.cpython-38.opt-1.pyc

U

e5d�~�@sdddgZddlZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddl
mZddl
mZm
Z
ddlmZd	Zd
ZdZdZe��Zd
d�Zdd�ZGdd�de�ZGdd�d�Zdd�ZGdd�de�Zd+dd�Zdd�ZGdd �d e�Z Gd!d�de!�Z"Gd"d#�d#e!�Z#e#Z$Gd$d%�d%e#�Z%Gd&d'�d'e!�Z&Gd(d)�d)e&�Z'Gd*d�de"�Z(dS),�Pool�
ThreadPool�N)�Empty�)�util)�get_context�TimeoutError)�wait�INIT�RUN�CLOSE�	TERMINATEcCstt|��S�N)�list�map��args�r�,/usr/lib64/python3.8/multiprocessing/pool.py�mapstar/srcCstt�|d|d��S)Nrr)r�	itertools�starmaprrrr�starmapstar2src@seZdZdd�Zdd�ZdS)�RemoteTracebackcCs
||_dSr��tb)�selfrrrr�__init__:szRemoteTraceback.__init__cCs|jSrr�rrrr�__str__<szRemoteTraceback.__str__N)�__name__�
__module__�__qualname__rrrrrrr9src@seZdZdd�Zdd�ZdS)�ExceptionWithTracebackcCs0t�t|�||�}d�|�}||_d||_dS)N�z

"""
%s""")�	traceback�format_exception�type�join�excr)rr)rrrrr@s
zExceptionWithTraceback.__init__cCst|j|jffSr)�rebuild_excr)rrrrr�
__reduce__Esz!ExceptionWithTraceback.__reduce__N)r r!r"rr+rrrrr#?sr#cCst|�|_|Sr)r�	__cause__)r)rrrrr*Hs
r*cs0eZdZdZ�fdd�Zdd�Zdd�Z�ZS)�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.cs.t|�|_t|�|_tt|��|j|j�dSr)�reprr)�value�superr-r)rr)r/��	__class__rrrTs

zMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r/r)rrrrrYs�zMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r2r rrrr�__repr__]szMaybeEncodingError.__repr__)r r!r"�__doc__rrr3�
__classcell__rrr1rr-Psr-rFc
Cs�|dk	r(t|t�r|dks(td�|���|j}|j}t|d�rR|j��|j	��|dk	rb||�d}|dks~|�r�||k�r�z
|�}	Wn(t
tfk
r�t�
d�Y�q�YnX|	dkr�t�
d��q�|	\}
}}}
}zd||
|�f}WnHtk
�r0}z(|�r|tk	�rt||j�}d|f}W5d}~XYnXz||
||f�WnRtk
�r�}z2t||d�}t�
d	|�||
|d|ff�W5d}~XYnXd}	}
}}}
}|d7}qft�
d
|�dS)NrzMaxtasks {!r} is not valid�_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks)�
isinstance�int�AssertionError�format�put�get�hasattrr6�close�_reader�EOFError�OSErrorr�debug�	Exception�_helper_reraises_exceptionr#�
__traceback__r-)�inqueue�outqueue�initializer�initargsZmaxtasks�wrap_exceptionr;r<Z	completed�task�job�i�funcr�kwds�result�e�wrappedrrr�workerasN�





�$
rScCs|�dS)z@Pickle-able helper function for use by _guarded_task_generation.Nr)ZexrrrrD�srDcs2eZdZdZdd��fdd�
Z�fdd�Z�ZS)�
_PoolCachez�
    Class that implements a cache for the Pool class that will notify
    the pool management threads every time the cache is emptied. The
    notification is done by the use of a queue that is provided when
    instantiating the cache.
    N��notifiercs||_t�j||�dSr)rVr0r)rrVrrOr1rrr�sz_PoolCache.__init__cs t��|�|s|j�d�dSr)r0�__delitem__rVr;)r�itemr1rrrW�sz_PoolCache.__delitem__)r r!r"r4rrWr5rrr1rrT�srTc@s�eZdZdZdZedd��ZdLdd�Zej	e
fd	d
�Zdd�Zd
d�Z
edd��Zedd��Zdd�Zedd��Zedd��Zdd�Zdd�Zdifdd�ZdMdd �ZdNd!d"�ZdOd#d$�Zd%d&�ZdPd(d)�ZdQd*d+�Zdiddfd,d-�ZdRd.d/�ZdSd0d1�ZedTd2d3��Ze d4d5��Z!ed6d7��Z"ed8d9��Z#ed:d;��Z$d<d=�Z%d>d?�Z&d@dA�Z'dBdC�Z(edDdE��Z)e dFdG��Z*dHdI�Z+dJdK�Z,dS)UrzS
    Class which supports an async version of applying functions to arguments.
    TcOs|j||�Sr��Process)�ctxrrOrrrrZ�szPool.ProcessNrcCsg|_t|_|pt�|_|��t��|_|j��|_	t
|j	d�|_||_||_
||_|dkrjt��phd}|dkrztd��|dk	r�t|�s�td��||_z|��WnHtk
r�|jD]}|jdkr�|��q�|jD]}|��q؂YnX|��}tjtj|j|j|j|j|j|j|j |j!|j
|j|j|j"||j	fd�|_#d|j#_$t%|j#_|j#�&�tjtj'|j|j(|j!|j|jfd�|_)d|j)_$t%|j)_|j)�&�tjtj*|j!|j+|jfd�|_,d|j,_$t%|j,_|j,�&�t-j.||j/|j|j |j!|j|j	|j#|j)|j,|jf	dd�|_0t%|_dS)	NrUrz&Number of processes must be at least 1zinitializer must be a callable��targetrT�)rZexitpriority)1�_poolr
�_stater�_ctx�
_setup_queues�queue�SimpleQueue�
_taskqueue�_change_notifierrT�_cache�_maxtasksperchild�_initializer�	_initargs�os�	cpu_count�
ValueError�callable�	TypeError�
_processes�_repopulate_poolrC�exitcode�	terminater(�_get_sentinels�	threadingZThreadr�_handle_workersrZ�_inqueue�	_outqueue�_wrap_exception�_worker_handler�daemonr�start�
_handle_tasks�
_quick_put�
_task_handler�_handle_results�
_quick_get�_result_handlerrZFinalize�_terminate_pool�
_terminate)r�	processesrHrI�maxtasksperchild�context�p�	sentinelsrrrr�s�





��
��
�
��z
Pool.__init__cCs>|j|kr:|d|��t|d�t|dd�dk	r:|j�d�dS)Nz&unclosed running multiprocessing pool )�sourcerf)r`�ResourceWarning�getattrrfr;)rZ_warnrrrr�__del__s

�zPool.__del__c	Cs0|j}d|j�d|j�d|j�dt|j��d�	S)N�<�.z state=z pool_size=�>)r2r!r"r`�lenr_)r�clsrrrr3sz
Pool.__repr__cCs|jjg}|jjg}||�Sr)rxr?rf)rZtask_queue_sentinelsZself_notifier_sentinelsrrrrts

zPool._get_sentinelscCsdd�|D�S)NcSsg|]}t|d�r|j�qS)�sentinel)r=r�)�.0rSrrr�
<listcomp>s
�z.Pool._get_worker_sentinels.<locals>.<listcomp>r�Zworkersrrr�_get_worker_sentinelss�zPool._get_worker_sentinelscCsPd}ttt|���D]6}||}|jdk	rt�d|�|��d}||=q|S)z�Cleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        FN�cleaning up worker %dT)�reversed�ranger�rrrrBr()�poolZcleanedrMrSrrr�_join_exited_workerss
zPool._join_exited_workerscCs0|�|j|j|j|j|j|j|j|j|j	|j
�
Sr)�_repopulate_pool_staticrarZrpr_rwrxrirjrhryrrrrrq.s�zPool._repopulate_poolc

Csft|t|��D]P}
||t||||||	fd�}|j�dd�|_d|_|��|�|�t�	d�qdS)z�Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        r\rZZ
PoolWorkerTzadded workerN)
r�r�rS�name�replacer{r|�appendrrB)r[rZr�r�rFrGrHrIr�rJrM�wrrrr�7s��
zPool._repopulate_pool_staticc

Cs*t�|�r&t�||||||||||	�
dS)zEClean up any exited workers and start replacements for them.
        N)rr�r�)
r[rZr�r�rFrGrHrIr�rJrrr�_maintain_poolJs
�zPool._maintain_poolcCs4|j��|_|j��|_|jjj|_|jjj|_	dSr)
rardrwrxr6�sendr~r?�recvr�rrrrrbVszPool._setup_queuescCs|jtkrtd��dS)NzPool not running)r`rrmrrrr�_check_running\s
zPool._check_runningcCs|�|||���S)zT
        Equivalent of `func(*args, **kwds)`.
        Pool must be running.
        )�apply_asyncr<)rrNrrOrrr�apply`sz
Pool.applycCs|�||t|���S)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        )�
_map_asyncrr<�rrN�iterable�	chunksizerrrrgszPool.mapcCs|�||t|���S)z�
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        )r�rr<r�rrrrnszPool.starmapcCs|�||t|||�S)z=
        Asynchronous version of `starmap()` method.
        )r�r�rrNr�r��callback�error_callbackrrr�
starmap_asyncvs�zPool.starmap_asyncc
csjz,d}t|�D]\}}||||fifVqWn8tk
rd}z||dt|fifVW5d}~XYnXdS)z�Provides a generator of tasks for imap and imap_unordered with
        appropriate handling for iterables which throw exceptions during
        iteration.���rN)�	enumeraterCrD)rZ
result_jobrNr�rM�xrQrrr�_guarded_task_generation~szPool._guarded_task_generationrcCs�|��|dkr:t|�}|j�|�|j||�|jf�|S|dkrPtd�|���t	�
|||�}t|�}|j�|�|jt|�|jf�dd�|D�SdS)zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        rzChunksize must be 1+, not {0:n}css|]}|D]
}|Vq
qdSrr�r��chunkrXrrr�	<genexpr>�szPool.imap.<locals>.<genexpr>N)r��IMapIteratorrer;r��_job�_set_lengthrmr:r�
_get_tasksr�rrNr�r�rP�task_batchesrrr�imap�s4�������z	Pool.imapcCs�|��|dkr:t|�}|j�|�|j||�|jf�|S|dkrPtd�|���t	�
|||�}t|�}|j�|�|jt|�|jf�dd�|D�SdS)zL
        Like `imap()` method but ordering of results is arbitrary.
        rzChunksize must be 1+, not {0!r}css|]}|D]
}|Vq
qdSrrr�rrrr��sz&Pool.imap_unordered.<locals>.<genexpr>N)r��IMapUnorderedIteratorrer;r�r�r�rmr:rr�rr�rrr�imap_unordered�s0������zPool.imap_unorderedcCs6|��t|||�}|j�|jd|||fgdf�|S)z;
        Asynchronous version of `apply()` method.
        rN)r��ApplyResultrer;r�)rrNrrOr�r�rPrrrr��szPool.apply_asynccCs|�||t|||�S)z9
        Asynchronous version of `map()` method.
        )r�rr�rrr�	map_async�s�zPool.map_asyncc
Cs�|��t|d�st|�}|dkrJtt|�t|j�d�\}}|rJ|d7}t|�dkrZd}t�|||�}t||t|�||d�}	|j	�
|�|	j||�df�|	S)zY
        Helper function to implement map, starmap and their async counterparts.
        �__len__N�rr�r�)
r�r=r�divmodr�r_rr��	MapResultrer;r�r�)
rrNr�Zmapperr�r�r�Zextrar�rPrrrr��s,
����zPool._map_asynccCs"t||d�|��s|��qdS)N)�timeout)r	�emptyr<)r��change_notifierr�rrr�_wait_for_updates�szPool._wait_for_updatescCspt��}|jtks |rX|jtkrX|�|||||||	|
||�
|�|�|
�}|�||�q|�d�t	�
d�dS)Nzworker handler exiting)ru�current_threadr`rr
r�r�r�r;rrB)r��cache�	taskqueuer[rZr�r�rFrGrHrIr�rJr�r��threadZcurrent_sentinelsrrrrv�s�
zPool._handle_workersc

Cspt��}t|jd�D]�\}}d}z�|D]�}|jtkrBt�d�q�z||�Wq&tk
r�}
zB|dd�\}	}z||	�	|d|
f�Wnt
k
r�YnXW5d}
~
XYq&Xq&|r�t�d�|r�|dnd}||d�W�qW�
�q
W5d}}}	Xqt�d�z6t�d�|�d�t�d	�|D]}|d��q.Wn tk
�r`t�d
�YnXt�d�dS)Nz'task handler found thread._state != RUN�Fzdoing set_length()rr�ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting)
rur��iterr<r`rrrBrC�_set�KeyErrorr;rA)
r�r;rGr�r�r�ZtaskseqZ
set_lengthrKrLrQ�idxr�rrrr}sB






zPool._handle_tasksc	Cs�t��}z
|�}Wn$ttfk
r6t�d�YdSX|jtkrNt�d�q�|dkrbt�d�q�|\}}}z||�||�Wnt	k
r�YnXd}}}q|�r@|jt
k�r@z
|�}Wn$ttfk
r�t�d�YdSX|dk�r�t�d�q�|\}}}z||�||�Wnt	k
�r0YnXd}}}q�t|d��r�t�d�z,td�D]}|j
���sv�q�|��q`Wnttfk
�r�YnXt�dt|�|j�dS)	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr?z"ensuring that outqueue is not full�
z7result handler exiting: len(cache)=%s, thread._state=%s)rur�rAr@rrBr`rr�r�r
r=r�r?�pollr�)rGr<r�r�rKrLrM�objrrrr�:s\











�zPool._handle_resultsccs0t|�}tt�||��}|s dS||fVqdSr)r��tupler�islice)rN�it�sizer�rrrr�vs
zPool._get_taskscCstd��dS)Nz:pool objects cannot be passed between processes or pickled)�NotImplementedErrorrrrrr+s�zPool.__reduce__cCs2t�d�|jtkr.t|_t|j_|j�d�dS)Nzclosing pool)rrBr`rrrzrfr;rrrrr>�s


z
Pool.closecCst�d�t|_|��dS)Nzterminating pool)rrBr
r`r�rrrrrs�s
zPool.terminatecCsjt�d�|jtkrtd��n|jttfkr4td��|j��|j	��|j
��|jD]}|��qXdS)Nzjoining poolzPool is still runningzIn unknown state)rrBr`rrmrr
rzr(rr�r_)rr�rrrr(�s






z	Pool.joincCs@t�d�|j��|��r<|j��r<|j��t�	d�qdS)Nz7removing tasks from inqueue until task handler finishedr)
rrBZ_rlock�acquire�is_aliver?r�r��time�sleep)rF�task_handlerr�rrr�_help_stuff_finish�s



zPool._help_stuff_finishc
CsXt�d�t|_|�d�t|_t�d�|�||t|��|��sXt|	�dkrXtd��t|_|�d�|�d�t�d�t	�
�|k	r�|��|r�t|dd�r�t�d�|D]}
|
j
dkr�|
��q�t�d�t	�
�|k	r�|��t�d	�t	�
�|k	�r|��|�rTt|dd��rTt�d
�|D](}
|
���r*t�d|
j�|
���q*dS)Nzfinalizing poolz&helping task handler/workers to finishrz.Cannot have cache with result_hander not alivezjoining worker handlerrszterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr�)rrBr
r`r;r�r�r�r9rur�r(r=rrrs�pid)r�r�rFrGr�r�Zworker_handlerr�Zresult_handlerr�r�rrrr��sB


�









zPool._terminate_poolcCs|��|Sr)r�rrrr�	__enter__�szPool.__enter__cCs|��dSr)rs)r�exc_typeZexc_valZexc_tbrrr�__exit__�sz
Pool.__exit__)NNrNN)N)N)NNN)r)r)NNN)NNN)N)-r r!r"r4ry�staticmethodrZr�warnings�warnrr�r3rtr�r�rqr�r�rbr�r�rrr�r�r�r�r�r�r�r��classmethodrvr}r�r�r+r>rsr(r�r�r�r�rrrrr�sx
�
P

	



�


�

�
�


-
;


5c@s@eZdZdd�Zdd�Zdd�Zddd	�Zdd
d�Zdd
�ZdS)r�cCs>||_t��|_tt�|_|j|_||_||_	||j|j<dSr)
r_ruZEvent�_event�next�job_counterr�rg�	_callback�_error_callback)rr�r�r�rrrr�s

zApplyResult.__init__cCs
|j��Sr)r�Zis_setrrrr�ready�szApplyResult.readycCs|��std�|���|jS)Nz{0!r} not ready)r�rmr:�_successrrrr�
successful�szApplyResult.successfulNcCs|j�|�dSr)r�r	�rr�rrrr	�szApplyResult.waitcCs,|�|�|��st�|jr"|jS|j�dSr)r	r�rr��_valuer�rrrr<�s
zApplyResult.getcCsZ|\|_|_|jr$|jr$|�|j�|jr<|js<|�|j�|j��|j|j=d|_dSr)	r�r�r�r�r��setrgr�r_�rrMr�rrrr�s

zApplyResult._set)N)N)	r r!r"rr�r�r	r<r�rrrrr��s	

	r�c@seZdZdd�Zdd�ZdS)r�cCshtj||||d�d|_dg||_||_|dkrNd|_|j��|j|j	=n||t
||�|_dS)Nr�Tr)r�rr�r��
_chunksize�_number_leftr�r�rgr��bool)rr�r��lengthr�r�rrrrs
�
zMapResult.__init__cCs�|jd8_|\}}|rv|jrv||j||j|d|j�<|jdkr�|jrZ|�|j�|j|j=|j��d|_	nL|s�|jr�d|_||_|jdkr�|j
r�|�
|j�|j|j=|j��d|_	dS)NrrF)r�r�r�r�r�rgr�r�r�r_r�)rrMZsuccess_result�successrPrrrr�$s&







zMapResult._setN)r r!r"rr�rrrrr�s
r�c@s:eZdZdd�Zdd�Zddd�ZeZdd	�Zd
d�ZdS)
r�cCsT||_t�t���|_tt�|_|j|_t	�
�|_d|_d|_
i|_||j|j<dS)Nr)r_ruZ	ConditionZLock�_condr�r�r�rg�collections�deque�_items�_index�_length�	_unsorted)rr�rrrrBs

zIMapIterator.__init__cCs|Srrrrrr�__iter__MszIMapIterator.__iter__NcCs�|j��z|j��}Wnztk
r�|j|jkr>d|_td�|j�|�z|j��}Wn2tk
r�|j|jkr�d|_td�t	d�YnXYnXW5QRX|\}}|r�|S|�dSr)
r�r��popleft�
IndexErrorr�rr_�
StopIterationr	r)rr�rXr�r/rrrr�Ps&zIMapIterator.nextc	Cs�|j��|j|krn|j�|�|jd7_|j|jkrb|j�|j�}|j�|�|jd7_q,|j��n
||j|<|j|jkr�|j|j	=d|_
W5QRXdS�Nr)r�r�r�r�r�pop�notifyrrgr�r_r�rrrr�hs


zIMapIterator._setc	CsB|j�2||_|j|jkr4|j��|j|j=d|_W5QRXdSr)r�rr�rrgr�r_)rr�rrrr�ys

zIMapIterator._set_length)N)	r r!r"rrr��__next__r�r�rrrrr�@s
r�c@seZdZdd�ZdS)r�c	CsV|j�F|j�|�|jd7_|j��|j|jkrH|j|j=d|_W5QRXdSr)	r�r�r�r�rrrgr�r_r�rrrr��s

zIMapUnorderedIterator._setN)r r!r"r�rrrrr��sr�c@sVeZdZdZedd��Zddd�Zdd	�Zd
d�Zedd
��Z	edd��Z
dd�ZdS)rFcOsddlm}|||�S)NrrY)ZdummyrZ)r[rrOrZrrrrZ�szThreadPool.ProcessNrcCst�||||�dSr)rr)rr�rHrIrrrr�szThreadPool.__init__cCs,t��|_t��|_|jj|_|jj|_dSr)rcrdrwrxr;r~r<r�rrrrrb�s


zThreadPool._setup_queuescCs
|jjgSr)rfr?rrrrrt�szThreadPool._get_sentinelscCsgSrrr�rrrr��sz ThreadPool._get_worker_sentinelscCsFz|jdd�qWntjk
r(YnXt|�D]}|�d�q2dS)NF)�block)r<rcrr�r;)rFr�r�rMrrrr��szThreadPool._help_stuff_finishcCst�|�dSr)r�r�)rr�r�r�rrrr��szThreadPool._wait_for_updates)NNr)r r!r"ryr�rZrrbrtr�r�r�rrrrr�s




)NrNF))�__all__r�rrkrcrur�r%r�rr$rrrZ
connectionr	r
rrr
�countr�rrrCrr#r*r-rSrD�dictrT�objectrr�ZAsyncResultr�r�r�rrrrr�<module>
sN	�
-=)+E
?>