Your IP : 3.143.212.252

Current Path : /proc/self/root/proc/self/root/lib64/python3.6/multiprocessing/__pycache__/
Upload File :
Current File : //proc/self/root/proc/self/root/lib64/python3.6/multiprocessing/__pycache__/pool.cpython-36.pyc

3


 \�e�@s8ddgZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddlm
Z
mZdZdZ
dZej�Zdd	�Zd
d�ZGdd
�d
e�ZGdd�d�Zdd�ZGdd�de�Zdfddfdd�Zdd�ZGdd�de�ZGdd�de�ZeZGdd�de�ZGdd�de�ZGd d!�d!e�Z Gd"d�de�Z!dS)#�Pool�
ThreadPool�N�)�util)�get_context�TimeoutError�cCstt|��S)N)�list�map)�args�r�,/usr/lib64/python3.6/multiprocessing/pool.py�mapstar+srcCsttj|d|d��S)Nrr)r	�	itertools�starmap)rrrr
�starmapstar.src@seZdZdd�Zdd�ZdS)�RemoteTracebackcCs
||_dS)N)�tb)�selfrrrr
�__init__6szRemoteTraceback.__init__cCs|jS)N)r)rrrr
�__str__8szRemoteTraceback.__str__N)�__name__�
__module__�__qualname__rrrrrr
r5src@seZdZdd�Zdd�ZdS)�ExceptionWithTracebackcCs0tjt|�||�}dj|�}||_d||_dS)N�z

"""
%s""")�	traceback�format_exception�type�join�excr)rr rrrr
r<s
zExceptionWithTraceback.__init__cCst|j|jffS)N)�rebuild_excr r)rrrr
�
__reduce__Asz!ExceptionWithTraceback.__reduce__N)rrrrr"rrrr
r;srcCst|�|_|S)N)r�	__cause__)r rrrr
r!Ds
r!cs0eZdZdZ�fdd�Zdd�Zdd�Z�ZS)�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.cs.t|�|_t|�|_tt|�j|j|j�dS)N)�reprr �value�superr$r)rr r&)�	__class__rr
rPs

zMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r&r )rrrr
rUszMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r(r)rrrr
�__repr__YszMaybeEncodingError.__repr__)rrr�__doc__rrr)�
__classcell__rr)r(r
r$Lsr$Fc'Cs�|dks t|�tkr|dks t�|j}|j}t|d�rJ|jj�|jj�|dk	rZ||�d}�xL|dksx|ot||k�r�y
|�}	Wn$t	t
fk
r�tjd�PYnX|	dkr�tjd�P|	\}
}}}
}yd||
|�f}WnFt
k
�r"}z(|o�|tk	�r
t||j�}d|f}WYdd}~XnXy||
||f�WnRt
k
�r�}z4t||d�}tjd|�||
|d|ff�WYdd}~XnXd}	}
}}}
}|d7}qbWtjd	|�dS)
Nr�_writerz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFrz0Possible encoding error while sending result: %szworker exiting after %d tasks)r�int�AssertionError�put�get�hasattrr,�close�_reader�EOFError�OSErrorr�debug�	Exception�_helper_reraises_exceptionr�
__traceback__r$)�inqueue�outqueue�initializer�initargsZmaxtasksZwrap_exceptionr/r0Z	completed�task�job�i�funcr�kwds�result�e�wrappedrrr
�worker]sF 





$rFcCs|�dS)z@Pickle-able helper function for use by _guarded_task_generation.Nr)Zexrrr
r8�sr8c@s6eZdZdZdZdd�Zddfddfdd�Zdd	�Zd
d�Zdd
�Z	dd�Z
fifdd�Zd=dd�Zd>dd�Z
d?dd�Zdd�Zd@dd�ZdAdd�Zfiddfdd �ZdBd!d"�ZdCd#d$�Zed%d&��Zed'd(��Zed)d*��Zed+d,��Zd-d.�Zd/d0�Zd1d2�Zd3d4�Zed5d6��Zed7d8��Z d9d:�Z!d;d<�Z"dS)DrzS
    Class which supports an async version of applying functions to arguments.
    TcOs|jj||�S)N)�_ctx�Process)rrrBrrr
rH�szPool.ProcessNcCsn|pt�|_|j�tj�|_i|_t|_||_	||_
||_|dkrPtj
�pNd}|dkr`td��|dk	rzt|�rztd��||_g|_|j�tjtj|fd�|_d|j_t|j_|jj�tjtj|j|j|j|j|jfd�|_d|j_t|j_|jj�tjtj|j|j |jfd�|_!d|j!_t|j!_|j!j�t"j#||j$|j|j%|j|j|j|j|j!|jfdd�|_&dS)Nrz&Number of processes must be at least 1zinitializer must be a callable)�targetrT�)rZexitpriority)'rrG�
_setup_queues�queue�Queue�
_taskqueue�_cache�RUN�_state�_maxtasksperchild�_initializer�	_initargs�os�	cpu_count�
ValueError�callable�	TypeError�
_processes�_pool�_repopulate_pool�	threadingZThreadr�_handle_workers�_worker_handler�daemon�start�
_handle_tasks�
_quick_put�	_outqueue�
_task_handler�_handle_results�
_quick_get�_result_handlerrZFinalize�_terminate_pool�_inqueue�
_terminate)r�	processesr<r=Zmaxtasksperchild�contextrrr
r�sT




z
Pool.__init__cCsZd}xPttt|j���D]:}|j|}|jdk	rtjd|�|j�d}|j|=qW|S)z�Cleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        FNzcleaning up worker %dT)�reversed�range�lenr[�exitcoderr6r)rZcleanedr@rFrrr
�_join_exited_workers�s

zPool._join_exited_workersc	Cs|xvt|jt|j��D]^}|jt|j|j|j|j	|j
|jfd�}|jj|�|j
jdd�|_
d|_|j�tjd�qWdS)z�Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        )rIrrHZ
PoolWorkerTzadded workerN)rorZrpr[rHrFrjrdrSrTrR�_wrap_exception�append�name�replacer`rarr6)rr@�wrrr
r\�szPool._repopulate_poolcCs|j�r|j�dS)zEClean up any exited workers and start replacements for them.
        N)rrr\)rrrr
�_maintain_pool�szPool._maintain_poolcCs4|jj�|_|jj�|_|jjj|_|jjj|_	dS)N)
rGZSimpleQueuerjrdr,�sendrcr3�recvrg)rrrr
rK�szPool._setup_queuescCs |jtkst�|j|||�j�S)z6
        Equivalent of `func(*args, **kwds)`.
        )rQrPr.�apply_asyncr0)rrArrBrrr
�apply�sz
Pool.applycCs|j||t|�j�S)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        )�
_map_asyncrr0)rrA�iterable�	chunksizerrr
r
szPool.mapcCs|j||t|�j�S)z�
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        )r}rr0)rrAr~rrrr
rszPool.starmapcCs|j||t|||�S)z=
        Asynchronous version of `starmap()` method.
        )r}r)rrAr~r�callback�error_callbackrrr
�
starmap_asyncszPool.starmap_asyncccsny0d}x&t|�D]\}}||||fifVqWWn8tk
rh}z||dt|fifVWYdd}~XnXdS)z�Provides a generator of tasks for imap and imap_unordered with
        appropriate handling for iterables which throw exceptions during
        iteration.rN���)�	enumerater7r8)rZ
result_jobrAr~r@�xrDrrr
�_guarded_task_generationszPool._guarded_task_generationrcCs�|jtkrtd��|dkrFt|j�}|jj|j|j||�|j	f�|S|dksRt
�tj|||�}t|j�}|jj|j|jt
|�|j	f�dd�|D�SdS)zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        zPool not runningrcss|]}|D]
}|Vq
qdS)Nr)�.0�chunk�itemrrr
�	<genexpr>@szPool.imap.<locals>.<genexpr>N)rQrPrW�IMapIteratorrOrNr/r��_job�_set_lengthr.r�
_get_tasksr)rrAr~rrC�task_batchesrrr
�imap's"




z	Pool.imapcCs�|jtkrtd��|dkrFt|j�}|jj|j|j||�|j	f�|S|dksRt
�tj|||�}t|j�}|jj|j|jt
|�|j	f�dd�|D�SdS)zL
        Like `imap()` method but ordering of results is arbitrary.
        zPool not runningrcss|]}|D]
}|Vq
qdS)Nr)r�r�r�rrr
r�[sz&Pool.imap_unordered.<locals>.<genexpr>N)rQrPrW�IMapUnorderedIteratorrOrNr/r�r�r�r.rr�r)rrAr~rrCr�rrr
�imap_unorderedBs"




zPool.imap_unorderedcCsB|jtkrtd��t|j||�}|jj|jd|||fgdf�|S)z;
        Asynchronous version of `apply()` method.
        zPool not runningrN)rQrPrW�ApplyResultrOrNr/r�)rrArrBr�r�rCrrr
r{]s

zPool.apply_asynccCs|j||t|||�S)z9
        Asynchronous version of `map()` method.
        )r}r)rrAr~rr�r�rrr
�	map_asynchszPool.map_asyncc
Cs�|jtkrtd��t|d�s$t|�}|dkrTtt|�t|j�d�\}}|rT|d7}t|�dkrdd}tj	|||�}t
|j|t|�||d�}	|jj
|j|	j||�df�|	S)zY
        Helper function to implement map, starmap and their async counterparts.
        zPool not running�__len__N�rr)r�)rQrPrWr1r	�divmodrpr[rr��	MapResultrOrNr/r�r�)
rrAr~Zmapperrr�r�Zextrar�rCrrr
r}ps&

zPool._map_asynccCsTtj�}x0|jtks$|jr8|jtkr8|j�tjd�q
W|j	j
d�tjd�dS)Ng�������?zworker handler exiting)
r]�current_threadrQrPrO�	TERMINATErx�time�sleeprNr/rr6)�pool�threadrrr
r^�szPool._handle_workersc
Csjtj�}x�t|jd�D]�\}}d}z�x�|D]�}|jr@tjd�Py||�Wq*tk
r�}	zD|dd�\}
}y||
j|d|	f�Wnt	k
r�YnXWYdd}	~	Xq*Xq*W|r�tjd�|r�|dnd}||d�wPWdd}}}
XqWtjd�y:tjd�|j
d�tjd�x|D]}|d��q&WWn tk
�rZtjd	�YnXtjd
�dS)Nz'task handler found thread._state != RUNrFzdoing set_length()rztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr�)r]r��iterr0rQrr6r7�_set�KeyErrorr/r5)
�	taskqueuer/r;r��cacher�ZtaskseqZ
set_lengthr>rDr?�idx�prrr
rb�sB







zPool._handle_taskscCs�tj�}x�y
|�}Wn"ttfk
r6tjd�dSX|jrX|jtksLt�tjd�P|dkrltjd�P|\}}}y||j	||�Wnt
k
r�YnXd}}}q
Wx�|o�|jtk�rJy
|�}Wn"ttfk
r�tjd�dSX|dk�rtjd�q�|\}}}y||j	||�Wnt
k
�r:YnXd}}}q�Wt|d��r�tjd�y,x&td�D]}|j
j��s�P|��qnWWnttfk
�r�YnXtjdt|�|j�dS)	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr3z"ensuring that outqueue is not full�
z7result handler exiting: len(cache)=%s, thread._state=%s)r]r�r5r4rr6rQr�r.r�r�r1ror3�pollrp)r;r0r�r�r>r?r@�objrrr
rf�s\










zPool._handle_resultsccs4t|�}x&ttj||��}|s"dS||fVq
WdS)N)r��tupler�islice)rA�it�sizer�rrr
r�szPool._get_taskscCstd��dS)Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)rrrr
r"szPool.__reduce__cCs&tjd�|jtkr"t|_t|j_dS)Nzclosing pool)rr6rQrP�CLOSEr_)rrrr
r2s

z
Pool.closecCs$tjd�t|_t|j_|j�dS)Nzterminating pool)rr6r�rQr_rk)rrrr
�	terminates
zPool.terminatecCsVtjd�|jttfkst�|jj�|jj�|j	j�x|j
D]}|j�qBWdS)Nzjoining pool)rr6rQr�r�r.r_rrerhr[)rr�rrr
rs



z	Pool.joincCsDtjd�|jj�x*|j�r>|jj�r>|jj�tj	d�qWdS)Nz7removing tasks from inqueue until task handler finishedr)
rr6Z_rlock�acquire�is_aliver3r�rzr�r�)r:�task_handlerr�rrr
�_help_stuff_finish(s



zPool._help_stuff_finishc	
CsFtjd�t|_t|_tjd�|j||t|��|j�sJt|�dksJt�t|_|jd�tjd�t	j
�|k	rx|j�|r�t|dd�r�tjd�x|D]}	|	j
dkr�|	j�q�Wtjd�t	j
�|k	r�|j�tjd�t	j
�|k	r�|j�|�rBt|dd��rBtjd	�x0|D](}	|	j��rtjd
|	j�|	j��qWdS)Nzfinalizing poolz&helping task handler/workers to finishrzjoining worker handlerr�zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)rr6r�rQr�rpr�r.r/r]r�rr1rqr��pid)
�clsr�r:r;r�Zworker_handlerr�Zresult_handlerr�r�rrr
ri1s8











zPool._terminate_poolcCs|S)Nr)rrrr
�	__enter___szPool.__enter__cCs|j�dS)N)r�)r�exc_typeZexc_valZexc_tbrrr
�__exit__bsz
Pool.__exit__)N)N)NNN)r)r)NNN)NNN)#rrrr*rsrHrrrr\rxrKr|r
rr�r�r�r�r{r�r}�staticmethodr^rbrfr�r"r2r�rr��classmethodrir�r�rrrr
r�sF8








.<			.c@s@eZdZdd�Zdd�Zdd�Zddd	�Zdd
d�Zdd
�ZdS)r�cCs4tj�|_tt�|_||_||_||_|||j<dS)N)	r]ZEvent�_event�next�job_counterr�rO�	_callback�_error_callback)rr�r�r�rrr
rks

zApplyResult.__init__cCs
|jj�S)N)r�Zis_set)rrrr
�readysszApplyResult.readycCs|j�st�|jS)N)r�r.�_success)rrrr
�
successfulvszApplyResult.successfulNcCs|jj|�dS)N)r��wait)r�timeoutrrr
r�zszApplyResult.waitcCs,|j|�|j�st�|jr"|jS|j�dS)N)r�r�rr��_value)rr�rrr
r0}s
zApplyResult.getcCsV|\|_|_|jr$|jr$|j|j�|jr>|jr>|j|j�|jj�|j|j=dS)N)r�r�r�r�r��setrOr�)rr@r�rrr
r��s
zApplyResult._set)N)N)	rrrrr�r�r�r0r�rrrr
r�is

	r�c@seZdZdd�Zdd�ZdS)r�cCsftj||||d�d|_dg||_||_|dkrLd|_|jj�||j=n||t	||�|_dS)N)r�Tr)
r�rr�r��
_chunksize�_number_leftr�r�r��bool)rr�r�lengthr�r�rrr
r�s


zMapResult.__init__cCs�|jd8_|\}}|rp|jrp||j||j|d|j�<|jdkr�|jrZ|j|j�|j|j=|jj�nH|r�|jr�d|_||_|jdkr�|j	r�|j	|j�|j|j=|jj�dS)NrrF)
r�r�r�r�r�rOr�r�r�r�)rr@Zsuccess_result�successrCrrr
r��s"




zMapResult._setN)rrrrr�rrrr
r��s
r�c@s:eZdZdd�Zdd�Zddd�ZeZdd	�Zd
d�ZdS)
r�cCsJtjtj��|_tt�|_||_tj	�|_
d|_d|_i|_
|||j<dS)Nr)r]Z	ConditionZLock�_condr�r�r�rO�collections�deque�_items�_index�_length�	_unsorted)rr�rrr
r�s

zIMapIterator.__init__cCs|S)Nr)rrrr
�__iter__�szIMapIterator.__iter__NcCs�|j�~y|jj�}Wnhtk
r~|j|jkr6t�|jj|�y|jj�}Wn(tk
rx|j|jkrpt�t�YnXYnXWdQRX|\}}|r�|S|�dS)N)	r�r��popleft�
IndexErrorr�r��
StopIterationr�r)rr�r�r�r&rrr
r��s"zIMapIterator.nextc
Cs�|j��|j|krr|jj|�|jd7_x8|j|jkrd|jj|j�}|jj|�|jd7_q.W|jj�n
||j|<|j|jkr�|j|j	=WdQRXdS)Nr)
r�r�r�rtr��pop�notifyr�rOr�)rr@r�rrr
r��s

zIMapIterator._setc	Cs<|j�,||_|j|jkr.|jj�|j|j=WdQRXdS)N)r�r�r�r�rOr�)rr�rrr
r��s

zIMapIterator._set_length)N)	rrrrr�r��__next__r�r�rrrr
r��s

r�c@seZdZdd�ZdS)r�c
CsP|j�@|jj|�|jd7_|jj�|j|jkrB|j|j=WdQRXdS)Nr)r�r�rtr�r�r�rOr�)rr@r�rrr
r�s
zIMapUnorderedIterator._setN)rrrr�rrrr
r��sr�c@s@eZdZdZedd��Zddffdd�Zdd�Zed	d
��ZdS)rFcOsddlm}|||�S)Nr)rH)ZdummyrH)rrBrHrrr
rHszThreadPool.ProcessNcCstj||||�dS)N)rr)rrlr<r=rrr
rszThreadPool.__init__cCs,tj�|_tj�|_|jj|_|jj|_dS)N)rLrMrjrdr/rcr0rg)rrrr
rKs


zThreadPool._setup_queuesc
Cs<|j�,|jj�|jjdg|�|jj�WdQRXdS)N)Z	not_emptyrL�clear�extendZ
notify_all)r:r�r�rrr
r�s
zThreadPool._help_stuff_finish)	rrrrsr�rHrrKr�rrrr
rs
)"�__all__r]rLrr�rUr�rrrrrrPr�r��countr�rrr7rrr!r$rFr8�objectrr�ZAsyncResultr�r�r�rrrrr
�<module>
s@	*\&)@
?>