Vq d dgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ddl
m Z m
Z
ddlmZ dZd Zd
Zd Z ej Zd Zd
Z G d de Z G d d Zd Z G d de Z d%dZ d Z G d de Z G d d e! Z" G d d e! Z#e#Z$ G d d e# Z% G d d!e! Z& G d" d#e& Z' G d$ de" Z(dS )&Pool
ThreadPool N )util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEc . t t | S N)listmapargss ;/opt/alt/python311/lib64/python3.11/multiprocessing/pool.pymapstarr / s T
c ^ t t j | d | d S )Nr r )r itertoolsstarmapr s r starmapstarr 2 s% !$q'4733 4 44r c e Zd Zd Zd ZdS )RemoteTracebackc || _ d S r tb)selfr s r __init__zRemoteTraceback.__init__: s
r c | j S r r r s r __str__zRemoteTraceback.__str__< s wr N)__name__
__module__ __qualname__r! r$ r r r r 9 s2 r r c e Zd Zd Zd ZdS )ExceptionWithTracebackc t j t | || }d | }|| _ d|z | _ d S )N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r1 r s r r! z ExceptionWithTraceback.__init__@ sB
'S 3
;
;
WWR[[ 2%r c , t | j | j ffS r ) rebuild_excr1 r r# s r
__reduce__z!ExceptionWithTraceback.__reduce__E s TXtw ///r N)r% r& r' r! r4 r( r r r* r* ? s2 & & &
0 0 0 0 0r r* c . t | | _ | S r )r __cause__)r1 r s r r3 r3 H s #B''CM Jr c . e Zd ZdZ fdZd Zd Z xZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
safely sent through the socket.c t | | _ t | | _ t t | | j | j d S r )reprr1 valuesuperr8 r! )r r1 r; __class__s r r! zMaybeEncodingError.__init__T sH 99%[[
$''004:FFFFFr c ( d| j d| j dS )NzError sending result: 'z '. Reason: '')r; r1 r# s r r$ zMaybeEncodingError.__str__Y s" =AZZZ=AXXXG Gr c ( d| j j d| dS )N)r= r% r# s r __repr__zMaybeEncodingError.__repr__] s !^ 4 4 4ddd;;r )r% r& r' __doc__r! r$ rC
__classcell__r= s @r r8 r8 P sh ' 'G G G G G
G G G< < < < < < >QZFFFFFF , Ca ! ! ! ! , , ,(F1I66G JI
Ca%)* + + + + + + + + , 4877s7V7d7TDQ 7
x
I,@,@8 J.:;;;;;sB &
B1 1%CC<
D
D?$D::D?E
F# =F F#c | )z@Pickle-able helper function for use by _guarded_task_generation.r( )exs r rV rV s
Hr c 2 e Zd ZdZdd fd
Z fdZ xZS )
_PoolCachez
Class that implements a cache for the Pool class that will notify
the pool management threads every time the cache is emptied. The
notification is done by the use of a queue that is provided when
instantiating the cache.
Nnotifierc H || _ t j |i | d S r )rm r< r! )r rm r rc r= s r r! z_PoolCache.__init__ s,
$'$'''''r c t | | s | j d d S d S r )r< __delitem__rm rM )r itemr= s r rp z_PoolCache.__delitem__ sK
D!!! $ M d # # # # # $ $r )r% r& r' rD r! rp rE rF s @r rk rk sj +/ ( ( ( ( ( ( (
$
$
$
$
$
$
$
$
$r rk c e Zd ZdZdZed Z d)dZej e
fdZ d Z d Z
ed
Zed Zd Zed
Zed Zd Zd Zdi fdZd*dZd*dZ d+dZd Zd,dZd,dZdi ddfdZ d+dZ d+dZ ed*d Z e d Z!ed Z"ed Z#ed Z$d! Z%d" Z&d# Z'd$ Z(ed% Z)e d& Z*d' Z+d( Z,dS )-r zS
Class which supports an async version of applying functions to arguments.
Tc | j |i |S r Process)ctxr rc s r ru z Pool.Process s s{D)D)))r Nr( c & g | _ t | _ |p
t | _ | t
j | _ | j | _ t | j | _ || _ || _
|| _ |t j pd}|dk rt# d |*t% |t& r|dk rt# d | t) | st+ d || _ | nR# t0 $ rE | j D ] }|j | | j D ]}| w xY w| }t; j t> j | j | j | j | j! | j | j | j" | j# | j
| j | j | j$ || j f | _% d| j% _& tN | j% _ | j% ( t; j t> j) | j | j* | j# | j | j f | _+ d| j+ _& tN | j+ _ | j+ ( t; j t> j, | j# | j- | j f | _. d| j. _& tN | j. _ | j. ( t_ j0 | | j1 | j | j" | j# | j | j | j% | j+ | j. | j f d
| _2 tN | _ d S ) Nrl r z&Number of processes must be at least 1r z/maxtasksperchild must be a positive int or Nonez initializer must be a callabletargetr T )r exitpriority)3_poolr
_stater _ctx
_setup_queuesqueue SimpleQueue
_taskqueue_change_notifierrk _cache_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorrI rJ callable TypeError
_processes_repopulate_poolrU exitcode terminater0 _get_sentinels threadingThreadr _handle_workersru _inqueue _outqueue_wrap_exception_worker_handlerdaemonr start
_handle_tasks
_quick_put
_task_handler_handle_results
_quick_get_result_handlerr Finalize_terminate_pool
_terminate)r processesrZ r[ maxtasksperchildcontextp sentinelss r r! z
Pool.__init__ s_
,{}} +-- !% 5 5 7 7 $*?@@@ !1 '! +!I q== EFF F ' .44
T8HA8M8M !RSSS "8K+@+@ " <== =# ! ! # # # # Z
"
":%KKMMMZ
'')) ( /'+t 4</4:t}dn#T^T5K& 43HJ '+ #&) # ""$$$ ' -%/4?DN*dk+
%)!$'! """ ( /'.$/4;? '+ #&) # ""$$$ - $&/4=$.$*')=t?Q& 5 s
"D7 7AFc | j |k rC |d| t | t | dd | j d d S d S d S )Nz&unclosed running multiprocessing pool )sourcer )r} ResourceWarninggetattrr rM )r _warnr s r __del__z Pool.__del__
sy ;# EC4CC!$
0
0
0
0t /66B%))$///// CBr c
p | j }d|j d|j d| j dt | j d S )NrA .z state=z pool_size=rB )r= r& r' r} lenr| )r clss r rC z
Pool.__repr__ sY n0CN 0 0S%5 0 0 0 0 __0 0 0 1r c B | j j g}| j j g}g ||S r )r rQ r )r task_queue_sentinelsself_notifier_sentinelss r r zPool._get_sentinels s1 $ 6 7 #'#8#@"A @%@(?@@r c d | D S )Nc < g | ]}t |d |j S )sentinel)rO r ).0rg s r
z.Pool._get_worker_sentinels.. s8 8 8 8F"6:668 8 8 8r r( workerss r _get_worker_sentinelszPool._get_worker_sentinels s# 8 88 8 8 8r c d}t t t | D ]A}| | }|j 0t j d|z | d}| |= B|S )zCleanup after any worker processes which have exited due to reaching
their specified lifetime. Returns True if any workers were cleaned up.
FNcleaning up worker %dT)reversedranger r r rT r0 )poolcleanedra rg s r _join_exited_workerszPool._join_exited_workers! st
%D **++ A!WF*
2Q6777
Gr c | | j | j | j | j | j | j | j | j | j | j
S r ) _repopulate_pool_staticr~ ru r r| r r r r r r r# s r r zPool._repopulate_pool1 sN ++DIt|,0O,0J
,0NDbI I I I I I I I I I Js !'
AA Ar c | |dk rLt | }| j | |j || |j f |S |dk r"t d | t
||| }t | }| j | |j t | |j f d |D S )zP
Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
r z Chunksize must be 1+, not {0:n}c 3 $ K | ] }|D ]}|V d S r r( r chunkrq s r z Pool.imap.. / ??U??D???????r ) r IMapIteratorr rM r _job _set_lengthr rL r
_get_tasksr r rb r r rd task_batchess r imapz Pool.imap s!
>>!$''F O 11&+tXNN&
M1}} 5<<!# #$ $ $ ??49EEL!$''F O 11&+292>@ @ &
@?f??? ?r c | |dk rLt | }| j | |j || |j f |S |dk r"t d | t
||| }t | }| j | |j t | |j f d |D S )zL
Like `imap()` method but ordering of results is arbitrary.
r z Chunksize must be 1+, not {0!r}c 3 $ K | ] }|D ]}|V d S r r( r s r r z&Pool.imap_unordered.. r r ) r IMapUnorderedIteratorr rM r r r r rL r r r r s r imap_unorderedzPool.imap_unordered s
>>*400F O 11&+tXNN&
M1}} 5<@ @ &
@?f??? ?r c | t | || }| j |j d|||fgdf |S )z;
Asynchronous version of `apply()` method.
r N)r ApplyResultr rM r )r rb r rc r r rd s r r zPool.apply_async sW
T8^<< v{AtT4 @ A4 HIII
r c @ | ||t ||| S )z9
Asynchronous version of `map()` method.
)r r r s r map_asynczPool.map_async s'
tXw 8 r c | t |d st | }|""" ;# DK*/D ' ! % %d + + + + + r c n t j d t | _ | d S )Nzterminating pool)r rT r
r} r r# s r r zPool.terminate s0
%&&& r c t j d | j t k rt d | j t
t fvrt d | j | j | j
| j D ]}| d S )Nz joining poolzPool is still runningzIn unknown state) r rT r} r r r r
r r0 r r r| )r r s r r0 z Pool.join s
>""" ;# 455 5
[ 2
2
2 /00 0 !!### !!! !!### A
FFHHHH r c z t j d | j | ru| j r`| j t j d | r | j Zd S d S d S d S )Nz7removing tasks from inqueue until task handler finishedr )
r rT _rlockacquireis_aliverQ r r timesleep)rX task_handlerr s r _help_stuff_finishzPool._help_stuff_finish s
LMMM ##%% '/*>*>*@*@ O " " " JqMMM ##%% '/*>*>*@*@ r c
t j d t |_ | d t |_ t j d | ||t
| | s"t
| dk rt d t |_ | d | d t j d t j
|ur| |rJt |d d r4t j d |D ] }
|
j
|
t j d t j
|ur| t j d t j
|ur| |rst |d d r_t j d
|D ]J}
|
r0t j d |
j z |
Gd S d S d S ) Nzfinalizing poolz&helping task handler/workers to finishr z.Cannot have cache with result_hander not alivezjoining worker handlerr zterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr )r rT r
r} rM r- r r) rK r r r0 rO r r pid) r r
rX rY r r worker_handlerr, result_handlerr r s r r zPool._terminate_pool sb
$%%%
!* D!!! '
;<<< w c$ii@@@ '')) BE
a @B B
B !* D!!! T
+,,, # % %^ ; ; ! ! ! "GDG[11 " J, - - -
"
":%KKMMM
)*** # % %\ 9 9
+,,, # % %^ ; ; ! ! ! GDG[11 J- . . .
::<< J 6 >???FFHHH
r c . | | S r )r r# s r __enter__zPool.__enter__ s r c . | d S r )r )r exc_typeexc_valexc_tbs r __exit__z
Pool.__exit__ s r )NNr( NNr )NNN)r )-r% r& r' rD r staticmethodru r! warningswarnr r rC r r r r r r r r r r r r r r r r r r r classmethodr r r r r4 rP r r0 r- r r3 r8 r( r r r r sa O* * \* CE04O O O Of %Ms 0 0 0 01 1 1 A A A
8 8 \8
\
B B B ' ' \'$ 9 9 \ 96 6 6 1 1 1 " 8 8 8 8I I I IM M M M FJ 9 9 9 9 J J J@ @ @ @:@ @ @ @8 &(b4 BF KO : " " " \"
- - [-( ++ ++ \++Z 9) 9) \9)v \
, , ,
\ 3 3 [3 j r c V e Zd Zd Zd Zd ZddZddZd Z e e
j Z dS ) r c || _ t j | _ t t
| _ |j | _ || _ || _ | | j | j < d S r )
r| r Event_eventnext job_counterr r _callback_error_callback)r r r r s r r! zApplyResult.__init__ sS
o'' %% k ! - !% DI r c 4 | j S r )r@ is_setr# s r readyzApplyResult.ready s {!!###r c | | s"t d | | j S )Nz{0!r} not ready)rG r rL _successr# s r
successfulzApplyResult.successful s7 zz|| = . 5 5d ; ;<< <} r Nc : | j | d S r )r@ r r r s r r zApplyResult.wait s !!!!!r c | | | st | j r| j S | j r )r rG r rI _valuerL s r rN zApplyResult.get sD 'zz|| = ; + r c |\ | _ | _ | j r!| j r| | j | j r!| j s| | j | j | j | j = d | _ d S r ) rI rN rC rD r@ setr r r| r ra r s r r zApplyResult._set s %("
t{ > (dm ( NN4; ' ' ' .
. - - - K "
r r )
r% r& r' r! rG rJ r rN r r< types GenericAlias__class_getitem__r( r r r r s & & &$ $ $
" " " " $ E$677r r c e Zd Zd Zd ZdS )r c t | ||| d| _ d g|z | _ || _ |dk r/d| _ | j | j | j = d S ||z t ||z z | _ d S )Nr Tr ) r r! rI rN
_chunksize _number_leftr@ rP r r bool)r r r lengthr r s r r! zMapResult.__init__ s T4,: < < <
fvo # >> !D KOO DI&&& & 1D)9K4L4L LD r c | xj dz c_ |\ }}|r| j r||| j || j z |dz | j z < | j dk rP| j r| | j | j | j = | j d | _ d S d S |s| j rd| _ || _ | j dk rP| j
r|
| j | j | j = | j d | _ d S d S )Nr r F) rX rI rN rW rC r r r@ rP r| rD )r ra success_resultsuccessrd s r r zMapResult._set) s1 Q ( "t} "CIDK$/)1Q3*?? @ A%%> 0NN4;///K * !!! !
&%
%t}
% %
$ A%%' 6((555K * !!! !
&%r N)r% r& r' r! r r( r r r r s5 M M M" " " " "r r c 2 e Zd Zd Zd ZddZeZd Zd ZdS )r c * || _ t j t j | _ t t | _ |j | _ t j
| _ d| _ d | _
i | _ | | j | j < d S )Nr )r| r ConditionLock_condrA rB r r collectionsdeque_items_index_length _unsorted)r r s r r! zIMapIterator.__init__G st
( )9)9::
%% k !')) !% DI r c | S r r( r# s r __iter__zIMapIterator.__iter__R s r Nc | j 5 | j }n# t $ r | j | j k rd | _ t d | j | | j }n5# t $ r( | j | j k rd | _ t d t d w xY wY nw xY wd d d n # 1 swxY w Y |\ }}|r|S |r )
rb re popleft
IndexErrorrf rg r|
StopIterationr r )r r rq r] r; s r rA zIMapIterator.nextU sE
Z 1 1
1{**,,
1
1
1;$,..!%DJ'T1
(((1 ;..00DD! 1 1 1{dl22%)
+5&D0 1 D
1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 L sJ C $C AB:(BB:2B44B:7C 9B::C C
C
c
| j 5 | j |k r| j | | xj dz
c_ | j | j v rW| j | j }| j | | xj dz
c_ | j | j v W| j n
|| j |< | j | j k r| j | j = d | _
d d d d S # 1 swxY w Y d S Nr ) rb rf re r rh popnotifyrg r r r| rQ s r r zIMapIterator._setm sE
Z " "{a ""3''' q kT^33 .,,T[99CK&&s+++KK1$KK kT^33
!!####$'q!{dl**K * !
" " " " " " " " " " " " " " " " " "s C#C88C<?C<c | j 5 || _ | j | j k r-| j | j | j = d | _ d d d d S # 1 swxY w Y d S r )rb rg rf rr r r r| )r rZ s r r zIMapIterator._set_length~ s
Z " "!DL{dl**
!!###K * !
" " " " " " " " " " " " " " " " " "s AAA !A r ) r% r& r' r! rj rA __next__r r r( r r r r E sk & & & , H" " """ " " " "r r c e Zd Zd ZdS )r c | j 5 | j | | xj dz
c_ | j | j | j k r| j | j = d | _ d d d d S # 1 swxY w Y d S rp ) rb re r rf rr rg r r r| rQ s r r zIMapUnorderedIterator._set s
Z " " K s # # # KK1 KK J {dl**K * !
" " " " " " " " " " " " " " " " " "s A(A==BBN)r% r& r' r r( r r r r s# " " " " "r r c n e Zd ZdZed Zd dZd Zd Zed Z ed Z
d
Z dS ) r Fc ddl m} ||i |S )Nr rt )dummyru )rv r rc ru s r ru zThreadPool.Process s) """"""w%%%%r Nr( c @ t | ||| d S r )r r! )r r rZ r[ s r r! zThreadPool.__init__ s
dI{H=====r c t j | _ t j | _ | j j | _ | j j | _ d S r )r r r r rM r rN r r# s r r zThreadPool._setup_queues s> )++
*,, -+ .,r c | j j gS r )r rQ r# s r r zThreadPool._get_sentinels s %-..r c g S r r( r s r r z ThreadPool._get_worker_sentinels s r c | d # t j $ r Y nw xY wt | D ]}| d d S )NTF)block)rN r Emptyr rM )rX r, r ra s r r- z ThreadPool._help_stuff_finish s{
) % (((
){ D t A KK s ,,c . t j | d S r )r* r+ )r r r r s r r z ThreadPool._wait_for_updates s
7r )NNr( ) r% r& r' r r9 ru r! r r r r- r r( r r r r s O& & \&> > > >- - - / / / \ \ r )Nr( NF))__all__rc r r r r r* r- rR r: r, r r r
connectionr r
r r r
countrB r r rU r r* r3 r8 rg rV dictrk objectr r AsyncResultr r r r r( r r r s <
' ' ' ' ' ' ' ' io 5 5 5 i 0 0 0 0 0 0 0 0 < < < < < < < <" GK +< +< +< +