Your IP : 18.191.198.245

Current Path : /opt/alt/python34/lib64/python3.4/multiprocessing/__pycache__/
Upload File :
Current File : //opt/alt/python34/lib64/python3.4/multiprocessing/__pycache__/pool.cpython-34.pyc

�
e f�a�@s�ddgZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddlm
Z
mZdZdZ
dZej�Zdd	�Zd
d�ZGdd
�d
e�ZGdd�d�Zdd�ZGdd�de�Zdfdddd�ZGdd�de�ZGdd�de�ZeZGdd�de�ZGdd�de�ZGdd�de�ZGd d�de�Z dS)!�Pool�
ThreadPool�N�)�util)�get_context�TimeoutError�cCstt|��S)N)�list�map)�args�r�9/opt/alt/python34/lib64/python3.4/multiprocessing/pool.py�mapstar+srcCsttj|d|d��S)Nrr)r	�	itertools�starmap)rrrr
�starmapstar.src@s(eZdZdd�Zdd�ZdS)�RemoteTracebackcCs
||_dS)N)�tb)�selfrrrr
�__init__6szRemoteTraceback.__init__cCs|jS)N)r)rrrr
�__str__8szRemoteTraceback.__str__N)�__name__�
__module__�__qualname__rrrrrr
r5src@s(eZdZdd�Zdd�ZdS)�ExceptionWithTracebackcCsDtjt|�||�}dj|�}||_d||_dS)N�z

"""
%s""")�	traceback�format_exception�type�join�excr)rr rrrr
r<s	zExceptionWithTraceback.__init__cCst|j|jffS)N)�rebuild_excr r)rrrr
�
__reduce__Asz!ExceptionWithTraceback.__reduce__N)rrrrr"rrrr
r;srcCst|�|_|S)N)r�	__cause__)r rrrr
r!Dsr!cs@eZdZdZ�fdd�Zdd�Zdd�Z�S)�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.csAt|�|_t|�|_tt|�j|j|j�dS)N)�reprr �value�superr$r)rr r&)�	__class__rr
rPszMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r&r )rrrr
rUs	zMaybeEncodingError.__str__cCsdt|�S)Nz<MaybeEncodingError: %s>)�str)rrrr
�__repr__YszMaybeEncodingError.__repr__)rrr�__doc__rrr*rr)r(r
r$Lsr$Fc'Cs|dks0t|�tkr*|dks0t�|j}|j}t|d�rn|jj�|jj�n|dk	r�||�nd}xx|dks�|r||kry
|�}	Wn&t	t
fk
r�tjd�PYnX|	dkrtjd�Pn|	\}
}}}
}yd||
|�f}WnLt
k
r}}z,|r_t||j�}nd|f}WYdd}~XnXy||
||f�Wnbt
k
r�}zBt||d�}tjd|�||
|d|ff�WYdd}~XnX|d7}q�Wtjd	|�dS)
Nr�_writerz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFrz0Possible encoding error while sending result: %szworker exiting after %d tasks)r�int�AssertionError�put�get�hasattrr,�close�_reader�EOFError�OSErrorr�debug�	Exceptionr�
__traceback__r$)�inqueue�outqueue�initializer�initargsZmaxtasksZwrap_exceptionr/r0Z	completed�task�job�i�funcr�kwds�result�e�wrappedrrr
�worker]sD0		

!


	,rEc@s�eZdZdZdZdd�Zddfdddd�Zdd	�Zd
d�Zdd
�Z	dd�Z
fidd�Zddd�Zddd�Z
ddddd�Zddd�Zddd�Zfidddd�Zddddd �Zdddd!d"�Zed#d$��Zed%d&��Zed'd(��Zed)d*��Zd+d,�Zd-d.�Zd/d0�Zd1d2�Zed3d4��Zed5d6��Zd7d8�Z d9d:�Z!dS);rzS
    Class which supports an async version of applying functions to arguments.
    TcOs|jj||�S)N)�_ctx�Process)rrrArrr
rG�szPool.ProcessNcCs,|pt�|_|j�tj�|_i|_t|_||_	||_
||_|dkrytj
�psd}n|dkr�td��n|dk	r�t|�r�td��n||_g|_|j�tjdtjd|f�|_d|j_t|j_|jj�tjdtjd|j|j|j|j|jf�|_d|j_t|j_|jj�tjdtjd|j|j |jf�|_!d|j!_t|j!_|j!j�t"j#||j$d|j|j%|j|j|j|j|j!|jfdd�|_&dS)	Nrz&Number of processes must be at least 1zinitializer must be a callable�targetrTZexitpriority�)'rrF�
_setup_queues�queue�Queue�
_taskqueue�_cache�RUN�_state�_maxtasksperchild�_initializer�	_initargs�os�	cpu_count�
ValueError�callable�	TypeError�
_processes�_pool�_repopulate_pool�	threadingZThreadr�_handle_workers�_worker_handler�daemon�start�
_handle_tasks�
_quick_put�	_outqueue�
_task_handler�_handle_results�
_quick_get�_result_handlerrZFinalize�_terminate_pool�_inqueue�
_terminate)r�	processesr;r<Zmaxtasksperchild�contextrrr
r�sT
							
		
		
		
z
Pool.__init__cCszd}xmttt|j���D]P}|j|}|jdk	r"tjd|�|j�d}|j|=q"q"W|S)z�Cleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        FNzcleaning up worker %dT)�reversed�range�lenrZ�exitcoderr6r)rZcleanedr?rErrr
�_join_exited_workers�s"

zPool._join_exited_workerscCs�x�t|jt|j��D]�}|jdtd|j|j|j|j	|j
|jf�}|jj|�|j
jdd�|_
d|_|j�tjd�qWdS)z�Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        rHrrGZ
PoolWorkerTzadded workerN)rnrYrorZrGrErircrRrSrQ�_wrap_exception�append�name�replacer_r`rr6)rr?�wrrr
r[�s#	
zPool._repopulate_poolcCs|j�r|j�ndS)zEClean up any exited workers and start replacements for them.
        N)rqr[)rrrr
�_maintain_pool�szPool._maintain_poolcCsL|jj�|_|jj�|_|jjj|_|jjj|_	dS)N)
rFZSimpleQueuerircr,�sendrbr3�recvrf)rrrr
rJ�szPool._setup_queuescCs.|jtkst�|j|||�j�S)z6
        Equivalent of `func(*args, **kwds)`.
        )rPrOr.�apply_asyncr0)rr@rrArrr
�apply�sz
Pool.applycCs|j||t|�j�S)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        )�
_map_asyncrr0)rr@�iterable�	chunksizerrr
r
�szPool.mapcCs|j||t|�j�S)z�
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        )r|rr0)rr@r}r~rrr
rszPool.starmapcCs|j||t|||�S)z=
        Asynchronous version of `starmap()` method.
        )r|r)rr@r}r~�callback�error_callbackrrr
�
starmap_asyncszPool.starmap_asyncrcs�|jtkrtd��n|dkrrt|j��|jj��fdd�t|�D��jf��S|dks�t	�t
j�||�}t|j��|jj�fdd�t|�D��jf�dd��D�SdS)zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        zPool not runningrc3s0|]&\}}�j|�|fifVqdS)N)�_job)�.0r?�x)r@rBrr
�	<genexpr>szPool.imap.<locals>.<genexpr>c3s0|]&\}}�j|t|fifVqdS)N)r�r)r�r?r�)rBrr
r�%scss"|]}|D]}|Vq
qdS)Nr)r��chunk�itemrrr
r�'sN)rPrOrV�IMapIteratorrNrMr/�	enumerate�_set_lengthr.r�
_get_tasks)rr@r}r~�task_batchesr)r@rBr
�imapsz	Pool.imapcs�|jtkrtd��n|dkrrt|j��|jj��fdd�t|�D��jf��S|dks�t	�t
j�||�}t|j��|jj�fdd�t|�D��jf�dd��D�SdS)zL
        Like `imap()` method but ordering of results is arbitrary.
        zPool not runningrc3s0|]&\}}�j|�|fifVqdS)N)r�)r�r?r�)r@rBrr
r�1sz&Pool.imap_unordered.<locals>.<genexpr>c3s0|]&\}}�j|t|fifVqdS)N)r�r)r�r?r�)rBrr
r�8scss"|]}|D]}|Vq
qdS)Nr)r�r�r�rrr
r�:sN)rPrOrV�IMapUnorderedIteratorrNrMr/r�r�r.rr�)rr@r}r~r�r)r@rBr
�imap_unordered)szPool.imap_unorderedcCsb|jtkrtd��nt|j||�}|jj|jd|||fgdf�|S)z;
        Asynchronous version of `apply()` method.
        zPool not runningN)rPrOrV�ApplyResultrNrMr/r�)rr@rrArr�rBrrr
rz<s
+zPool.apply_asynccCs|j||t|||�S)z9
        Asynchronous version of `map()` method.
        )r|r)rr@r}r~rr�rrr
�	map_asyncGszPool.map_asyncc	s|jtkrtd��nt|d�s<t|�}n|dkr�tt|�t|j�d�\}}|r�|d7}q�nt|�dkr�d}ntj	|||�}t
|j|t|�|d|��|jj
��fdd	�t|�D�df��S)
zY
        Helper function to implement map, starmap and their async counterparts.
        zPool not running�__len__N�rrr�c3s0|]&\}}�j|�|fifVqdS)N)r�)r�r?r�)�mapperrBrr
r�csz"Pool._map_async.<locals>.<genexpr>)rPrOrVr1r	�divmodrorZrr��	MapResultrNrMr/r�)	rr@r}r�r~rr�Zextrar�r)r�rBr
r|Os (		zPool._map_asynccCsrtj�}xB|jtks6|jrP|jtkrP|j�tjd�qW|j	j
d�tjd�dS)Ng�������?zworker handler exiting)
r\�current_threadrPrOrN�	TERMINATErw�time�sleeprMr/rr6)�pool�threadrrr
r]gs*
zPool._handle_workerscCs>tj�}x�t|jd�D]�\}}d}d}	y�x�t|�D]�\}	}|jrmtjd�Pny||�WqGtk
r�}
zN|dd�\}}y||j	|d|
f�Wnt
k
r�YnXWYdd}
~
XqGXqGW|rtjd�||	d�nwPWqtk
r�}
zx|rD|dd�nd
\}}||kr}||j	|dd|
f�n|r�tjd�||	d�nWYdd}
~
XqXqWtjd�yFtjd�|jd�tjd	�x|D]}|d�q�WWntk
r,tjd
�YnXtjd�dS)Nrz'task handler found thread._state != RUNrFzdoing set_length()rztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting���)rr)
r\r��iterr0r�rPrr6r7�_set�KeyErrorr/r5)�	taskqueuer/r:r��cacher�ZtaskseqZ
set_lengthr=r?rCr>ZindZex�prrr
ratsN	


"!
(





zPool._handle_taskscCstj�}x�y
|�}Wn)ttfk
rGtjd�dSYnX|jrw|jtksft�tjd�Pn|dkr�tjd�Pn|\}}}y||j	||�Wqt
k
r�YqXqWx�|r�|jtkr�y
|�}Wn)ttfk
r$tjd�dSYnX|dkrDtjd�q�n|\}}}y||j	||�Wq�t
k
rYq�Xq�Wt|d�r�tjd�y5x.td�D] }|j
j�s�Pn|�q�WWq�ttfk
r�Yq�Xntjdt|�|j�dS)	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr3z"ensuring that outqueue is not full�
z7result handler exiting: len(cache)=%s, thread._state=%s)r\r�r5r4rr6rPr�r.r�r�r1rnr3�pollro)r:r0r�r�r=r>r?�objrrr
re�sX

		


	

	

	
	zPool._handle_resultsccsDt|�}x1ttj||��}|s1dS||fVqWdS)N)r��tupler�islice)r@�it�sizer�rrr
r��szPool._get_taskscCstd��dS)Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)rrrr
r"�szPool.__reduce__cCs8tjd�|jtkr4t|_t|j_ndS)Nzclosing pool)rr6rPrO�CLOSEr^)rrrr
r2�s
	z
Pool.closecCs0tjd�t|_t|j_|j�dS)Nzterminating pool)rr6r�rPr^rj)rrrr
�	terminate�s
	zPool.terminatecCsqtjd�|jttfks(t�|jj�|jj�|j	j�x|j
D]}|j�qYWdS)Nzjoining pool)rr6rPr�r�r.r^rrdrgrZ)rr�rrr
r�s



z	Pool.joincCsZtjd�|jj�x9|j�rU|jj�rU|jj�tj	d�qWdS)Nz7removing tasks from inqueue until task handler finishedr)
rr6Z_rlock�acquire�is_aliver3r�ryr�r�)r9�task_handlerr�rrr
�_help_stuff_finishs



zPool._help_stuff_finishc	
Cs�tjd�t|_t|_tjd�|j||t|��|j�sit|�dksit�t|_|jd�tjd�t	j
�|k	r�|j�n|rt|dd�rtjd�x-|D]"}	|	j
dkr�|	j�q�q�Wntjd�t	j
�|k	r-|j�ntjd�t	j
�|k	rY|j�n|r�t|dd�r�tjd	�x>|D]3}	|	j�r�tjd
|	j�|	j�q�q�WndS)Nzfinalizing poolz&helping task handler/workers to finishrzjoining worker handlerr�zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)rr6r�rPr�ror�r.r/r\r�rr1rpr��pid)
�clsr�r9r:r�Zworker_handlerr�Zresult_handlerr�r�rrr
rh
s8
		
$	










zPool._terminate_poolcCs|S)Nr)rrrr
�	__enter__;szPool.__enter__cCs|j�dS)N)r�)r�exc_typeZexc_valZexc_tbrrr
�__exit__>sz
Pool.__exit__)"rrrr+rrrGrrqr[rwrJr{r
rr�r�r�rzr�r|�staticmethodr]rarer�r"r2r�rr��classmethodrhr�r�rrrr
r�sD	8	

3:			.c@s^eZdZdd�Zdd�Zdd�Zddd	�Zdd
d�Zdd
�ZdS)r�cCsJtj�|_tt�|_||_||_||_|||j<dS)N)	r\ZEvent�_event�next�job_counterr�rN�	_callback�_error_callback)rr�rr�rrr
rGs			zApplyResult.__init__cCs
|jj�S)N)r�Zis_set)rrrr
�readyOszApplyResult.readycCs|j�st�|jS)N)r�r.�_success)rrrr
�
successfulRszApplyResult.successfulNcCs|jj|�dS)N)r��wait)r�timeoutrrr
r�VszApplyResult.waitcCs?|j|�|j�s"t�n|jr2|jS|j�dS)N)r�r�rr��_value)rr�rrr
r0Ys
		zApplyResult.getcCs{|\|_|_|jr7|jr7|j|j�n|jr]|jr]|j|j�n|jj�|j|j=dS)N)r�r�r�r�r��setrNr�)rr?r�rrr
r�bs
zApplyResult._set)	rrrrr�r�r�r0r�rrrr
r�Es	r�c@s(eZdZdd�Zdd�ZdS)r�cCs�tj|||d|�d|_dg||_||_|dkrjd|_|jj�||j=n||t	||�|_dS)Nr�Tr)
r�rr�r��
_chunksize�_number_leftr�r�r��bool)rr�r~�lengthrr�rrr
rss			

zMapResult.__init__cCs�|\}}|r�||j||j|d|j�<|jd8_|jdkr�|jrq|j|j�n|j|j=|jj�q�nHd|_||_|j	r�|j	|j�n|j|j=|jj�dS)NrrF)
r�r�r�r�rNr�r�r�r�r�)rr?Zsuccess_result�successrBrrr
r��s%	
			
zMapResult._setN)rrrrr�rrrr
r�qs
r�c@sUeZdZdd�Zdd�Zddd�ZeZdd	�Zd
d�ZdS)r�cCsktjtj��|_tt�|_||_tj	�|_
d|_d|_i|_
|||j<dS)Nr)r\Z	ConditionZLock�_condr�r�r�rN�collections�deque�_items�_index�_length�	_unsorted)rr�rrr
r�s				zIMapIterator.__init__cCs|S)Nr)rrrr
�__iter__�szIMapIterator.__iter__NcCs�|jj�z�y|jj�}Wn�tk
r�|j|jkrNt�n|jj|�y|jj�}Wn3tk
r�|j|jkr�t�nt	�YnXYnXWd|jj
�X|\}}|r�|S|�dS)N)r�r�r��popleft�
IndexErrorr�r��
StopIterationr�r�release)rr�r�r�r&rrr
r��s&

	
	zIMapIterator.nextc
Cs�|jj�z�|j|kr�|jj|�|jd7_xJ|j|jkr�|jj|j�}|jj|�|jd7_qAW|jj�n
||j|<|j|jkr�|j	|j
=nWd|jj�XdS)Nr)r�r�r�r�rsr��pop�notifyr�rNr�r�)rr?r�rrr
r��s

zIMapIterator._setc
Cs^|jj�z<||_|j|jkrH|jj�|j|j=nWd|jj�XdS)N)r�r�r�r�r�rNr�r�)rr�rrr
r��s
	
zIMapIterator._set_length)	rrrrr�r��__next__r�r�rrrr
r��s
r�c@seZdZdd�ZdS)r�c
Cst|jj�zR|jj|�|jd7_|jj�|j|jkr^|j|j=nWd|jj	�XdS)Nr)
r�r�r�rsr�r�r�rNr�r�)rr?r�rrr
r��s

zIMapUnorderedIterator._setN)rrrr�rrrr
r��sr�c@s[eZdZdZedd��Zddfdd�Zdd�Zed	d
��ZdS)rFcOsddlm}|||�S)Nr)rG)ZdummyrG)rrArGrrr
rG�szThreadPool.ProcessNcCstj||||�dS)N)rr)rrkr;r<rrr
r�szThreadPool.__init__cCs@tj�|_tj�|_|jj|_|jj|_dS)N)rKrLrircr/rbr0rf)rrrr
rJ�szThreadPool._setup_queuesc
CsW|jj�z5|jj�|jjdg|�|jj�Wd|jj�XdS)N)Z	not_emptyr�rK�clear�extendZ
notify_allr�)r9r�r�rrr
r�s

zThreadPool._help_stuff_finish)	rrrrrr�rGrrJr�rrrr
r�s
)!�__all__r\rKrr�rTr�rrrrrrOr�r��countr�rrr7rrr!r$rE�objectrr�ZAsyncResultr�r�r�rrrrr
�<module>
s:		,��&%I
?>