fjmsbdZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z mZddlmZmZddlmZddlmZddlmZdd lmZmZdd lmZmZd d lmZ m!Z!dd l"m#Z$ddl%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,d dl-m.Z.m/Z/m0Z0d dl1m2Z2m3Z3m4Z4d dl5m6Z6d dl7m8Z8d dl9m:Z:m;Z;d dlm?Z?GddZ@GddZAGdde@eAZBGdde@ZCGd d!e@eAZDGd"d#e@ZEGd$d%e@ZFGd&d'e@ZGGd(d)e@ZHdS)*zA This module contains class implementing MongoDB API interaction N)partial)ListAnyIterable)SessionResponse) HTTPAdapter)RequestException)Retry)Schema SchemaError)is_feature_visibleis_feature_hidden_server_wide) supportedget_advice_instance)gettext)detailed_advice_schemauser_sites_info_schemaadvice_list_schema) is_wp_path)docroot) api_serverprotoadviser_api_server) XRayAPIErrorXRayAPIEmptyResponseTaskNotFoundError)open_local_storage)Task)get_xray_exec_useruser_mode_verificationread_jwt_token)filter_by_non_existencec ,eZdZdZdddedefdZdZdZed efd Z ed e fd Z d efd Z dde ded efdZ dde ded e fdZ ddede dededed e f dZ ddededefdZd ded e fdZ d!deded efdZd ded e fdZdZdS)"Clientz Base client class unavailabletracing_task_id system_idr*c||_||_tjd|_t dt ddgt gdd}t|}t|_ |j td|t|j j d |j _ dS) N api_clientGETPOST)iii)totalallowed_methodsstatus_forcelistbackoff_factor) max_retries:// )timeout)r+task_idlogging getLoggerloggerr frozensetr rsessionmountrrrequest)selfr+r* retry_confadapters P/opt/cloudlinux/venv/lib64/python3.11/site-packages/xray/apiclient/api_client.py__init__zClient.__init__9s"& ' 55 +4eV_+E+E,5ooo,F,F*+--- *555yy  e==='222&t|';RHHH c>|jjd|jd|jS)N::z::tracing_task_id=) __class____name__ main_endpointr9rAs rD__repr__zClient.__repr__Hs*.)aaT-?aaSWS_aaarFc.|jjd|jS)NrH)rIrJr9rLs rD__str__zClient.__str__Ks.);;T\;;;rFreturnc:ttd)z Base endpoint z$instances are to set their endpoint!)NotImplementedError_rLs rDrKzClient.main_endpointNs "!$J"K"KLLLrFc"dtiS)zL Updated request headers :return: dict with headers zX-Authr#rLs rD_headerszClient._headersUs n&&  rFc d|ji}|r||tj|S)ze Construct query string :return: string including system_id and given kwargs r+)r+updateurllibparse urlencode)rAkwargsinitials rD _query_stringzClient._query_string_s@ /  # NN6 " " "|%%g...rFTresponsewith_status_checkc|jsY|jd|j|jd|j}t td|zd|ji|jd|j|j|j |jdkr| }ndd i}|r8|dd kr,t td t|z|S) z Perform preprocessing checks of received Response: - is it OK, e.g. 200 OK - was it successful, e.g. status == ok and extract JSON from the Response object :return: full JSON representation of response zServer responded: %s:z#Unable to connect to server with %s resp_dataextraz[%s:%s] Response received %sstatusokz"Received unsuccessful response: %s) rgr<debugtext status_codereasonrrSinfourljsonstr)rAr^r_ request_dataresults rD_preprocess_responsezClient._preprocess_responseis { + K  4hm D D D&2FFX_FFL788<G"HM2444 4 K  ;%18?%\ + + +  3 & &]]__FF%F  Zh4''"1%I#J#JSQW[[#XYYY rFc|||} |d}|jd|j||t ||S#t $rt cYSwxYw)z Check received response :param response: a requests.Response object :return: 'result' dict from JSON representation of response r_rqz[%s] Received response data %sN)rq)rrr<rlrjrKeyErrordict)rAr^r_rqdatas rD_process_responsezClient._process_responses**8=N+PP (#D K  =%14 9 9 9|*&9999K   66MMM spostr _give_xray_exceptionrSrx) rAryrzr{r_r|rrespes rD_postz Client._posts. 5! # # #  C C K  8' B B B a -|((7(CC|((.5wPR)TT a a a++Ax,-.W,X,XZZ_` a a%%d8I&KK KsA B B;&B66B;c |jd| |j||j}n8#t $r+}|||dtd|d}~wwxYw|||S)NzSending DELETE request to %sr~z DELETE failedz)Failed to DELETE data to X-Ray API serverrt) r<rlr>deleterUr rrSrx)rAryr{r_rrs rD_deletezClient._deletes 7! # # # c<&&x&GGDD c c c++Ax,-.Y,Z,Z\\ab c c%%d8I&KK Ks!? A4 &A//A4c 2||jd|}|jd| |j||j}n8#t$r+}|||dtd|d}~wwxYw|S)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: a requests Response object N?zSending GET request to %sr~z GET failedz(Failed to GET data from X-Ray API server) rKr]r<rlr>getrUr rrSrAryrrs rD_raw_getzClient._raw_gets  ,EEt/A/A/C/CEEH 4h??? b<##Hdm#DDDD b b b++Ax,-.X,Y,Y[[`a b b s!A B)&BBcZ||}|||S)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: full dict from JSON representation of response without any processing rt)rrr)rAryr_rs rD _get_fullzClient._get_fulls:}}X&&((;L)NN NrFc||} ||S#t$r}t|j|d}~wwxYw)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: 'result' dict from JSON representation of response )r9N)rrxrrr9rs rD_getz Client._getsl }}X&& ))$// /#   #   s, AA  Ac|jd||d|i |jdj}n#tt f$r|}YnwxYw|d|}t td|zS)z Process received exception :param exc: original exception :param api_endpoint: requested endpoint :param log_message: text for logging the error :param exc_message: text for internal exception z %s with %sryrcrz: z%s. Please, try again later)r<errorargsrk IndexErrorAttributeErrorrrS)rAexc api_endpoint log_message exc_messageexc_infoexception_datas rDrzClient._give_xray_exceptions , S!+\ :  < < < x{)HHN+   HHH (55855 + , ,~ =?? ?s5A  A T)NTTT)TTN)NT)rJ __module__ __qualname____doc__rorErMrOpropertyrKrvrUr]rboolrrrrxrrrrrrrFrDr'r'4s )6 I I IS I"% I I I Ibbb<<<MsMMMXM  $   X /////8<X04@C@59(-1=A&KOBFKKcKDK4K!%K;?KKOKKKK<7;*. K K Kt K#' K K K K )-,0 N N# N%) N58 N N N N  S D    ?????rFr'c>eZdZdZedefdZdedefdZ dS) TaskMixinz1 A mixin class with Task related methods rPcdS)z( Limit processed fields ) rmrf client_ip tracing_by tracing_count starttime ini_location initial_count request_count auto_taskuserrrLs rD task_fieldszTaskMixin.task_fields s %%rF dict_viewcrfd|D}|d|d<tdi|S)z@ Turn dictionary structure into valid Task type c.i|]\}}|jv||Srr.0kvrAs rD z#TaskMixin._task..s6,,,da$******rFr*r9r)itemsr )rAr task_views` rD_taskzTaskMixin._tasksX,,,,ioo&7&7,,, ():; )  i   rFN) rJrrrrtuplerrvr rrrFrDrrsg%U%%%X%!t!!!!!!!rFrc`eZdZdZedefdZdeffd Zdee ffd Z xZ S) TasksClientz! 'tasks' endpoint client rPc&tdtdS)z& Base endpoint: tasks r6z/api/xray/tasksrrrLs rDrKzTasksClient.main_endpoint!s 77J7777rFc HtdS)z Construct query string. Aimed to get auto tasks only :return: string including system_id and type=auto autotype)superr]rAr[rIs rDr]zTasksClient._query_string(s ww$$&$111rFcbt}fd|DS)z# Get list of Tasks c:g|]}|Sr)r)ritemrAs rD z)TasksClient.get_tasks..5s%222T 4  222rF)rr)rArwrIs` rD get_taskszTasksClient.get_tasks0s1ww||~~2222T2222rF) rJrrrrrorKr]rr r __classcell__rIs@rDrrs8s888X8 222222234:3333333333rFrceZdZdZdddedeffdZdZedd Zd e dd fd Z de fdZ edefdZ de fdZde defdZdedd fdZdedd fdZddZddZxZS) DBMClientzD Client class using local dbm storage instead of remote API r(r)r+r*ct||d|_||_dS)N)r+r*)rrE task_object_db_open_db)rAr+r*rIs rDrEzDBMClient.__init__=s; 9oNNN==??rFc8|jdSr)rcloserLs rD__del__zDBMClient.__del__Cs rFrP gdbm objectc,tjddS)zC Open dbm DB :return: corresponding object z/root/local_mongoc)dbmopenrrFrDrzDBMClient._db_openFs x+S111rF post_dataNcHtj||j|j<dS)z3 Update a DBM task with given data N)rndumpsrr9)rArs rDrzDBMClient._postNs""&I!6!6rFc tj|j|jS#t tjf$r"}ttd|d}~wwxYw)z: Get saved DBM data :return: dict zFailed to load taskN) rnloadsrr9decoderuJSONDecodeErrorrrS)rArs rDrzDBMClient._getTsq  @:dht|4;;==>> >$./ @ @ @q!67788a ? @s58A0A++A0c2tjjSr)uuiduuid1hexrrFrD_idz DBMClient._id^sz||rFcn|}|j|d<tdi||_|jS)1 Get saved task :return: r9r)rr9r r)rA saved_tasks rDget_taskzDBMClient.get_taskbs< YY[[ $ 9--*--rFtaskc||_|j|_d|_||||_|jS)a: Create new task and get unique ID url --> URL client_ip --> IP tracing_by --> time|request_qty tracing_count --> COUNT ini_location --> PATH status --> processing :param task: a Task instance :return: task ID hold)rr9rfras_dictrrArs rDcreatezDBMClient.createlsMxxzz |   4<<>>"""|rFrc|j|d|j_||j_||jdS)z Update started|continued task status --> running starttime --> new timestamp :return: Nrunning)rrrfrrrrArs rDrWzDBMClient.updatesW   # MMOOO"+%." 4#++--.....rFcountc|j|d|j_||j_||jdS)} Update stopped task status --> stopped tracing_count --> new value :return: Nstopped)rrrfrrrrArs rDstopzDBMClient.stopsW   # MMOOO"+).& 4#++--.....rFc|j|d|j_||jdS)Y Complete tracing task status --> completed :return: N completed)rrrfrrrLs rDcompletezDBMClient.completesL   # MMOOO"- 4#++--.....rFc |j|j=dSz6 Delete tracing task :return: N)rr9rLs rDrzDBMClient.deletes HT\ " " "rF)rPrrPN)rJrrrrorEr staticmethodrrvrrrr rrintrWrrrrrs@rDrr8s )6###S#"%###### 222\27t77777 @d@@@@    \  $    4C& / / / / / / /# /$ / / / / / / / /########rFrceZdZdZedefdZdeffd ZdeddfdZ deddfdZ dd Z dd Z e defd Zd edefd ZdeddfdZdeddfdZdddededdfdZdeddfdZddZddZddZxZS) APIClientz% X-Ray task API client class rPc&tdtdS)z% Base endpoint: task r6z/api/xray/taskrrLs rDrKzAPIClient.main_endpoints 66J6666rFc |jdkr't|jStS)z Construct query string :return: string either including system_id and task_id or system_id only r(r))r9rr]rs rDr]zAPIClient._query_stringsE <= ( (77(((FF Fww$$&&&rFrNc|jd|}||d|D}|d|_dS)z POST request to "create a task" API endpoint with given data Obtains a task ID :param post_data: dict with POST data z/create?c&i|]\}}|dk ||S)rrrrrs rDrz*APIClient._post_create..s2$6$6$6TQ$%$4$4%&q$4$4$4rFr*N)rKr]rrr9)rArry response_datas rD _post_createzAPIClient._post_createss (HH$2D2D2F2FHH 8$6$6Y__5F5F$6$6$677 %%67 rFcn|jd|}|||dS)z| POST request to "update a task" API endpoint with given data :param post_data: dict with POST data z/update?NrKr]r)rArrys rD _post_updatezAPIClient._post_updates@ (HH$2D2D2F2FHH 8Y'''''rFc|jdd}|d|}||dS)z< GET request to "share a task" API endpoint Nz/share-request?rKr]r)rAshare_endpointrys rD_sharezAPIClient._sharesL+CRC0$KKT5G5G5I5IKK (rFcl|jd|}||dS)z> POST request to "delete a task" API endpoint z/delete?Nr)rArys rDrzAPIClient._deletes>(HH$2D2D2F2FHH 8rFcP||S)r)rrrLs rDrzAPIClient.get_tasks zz$))++&&&rFrcd|_fd|DjS)aH Create new task and get unique ID url --> URL client_ip --> IP tracing_by --> time|request_qty tracing_count --> COUNT ini_location --> PATH status --> processing :param task: a Task instance :return: updated Task instance rc.i|]\}}|jv||Srrrs rDrz$APIClient.create..s6222DAq 000a000rF)rfrrrr9rs` rDrzAPIClient.createse  2222DLLNN,@,@,B,B222 3 3 3|rFrc6|d|ddS)z Update started|continued task status --> running starttime --> new timestamp :param starttime: time of starting the Task r)rfrNrrs rDrWzAPIClient.updates5 Y(133 4 4 4 4 4rFrc4|d|idS)zw Update tracing_count only. No status updated tracing_count --> new value :return: rNrrs rDupdate_count_onlyzAPIClient.update_count_onlys# ?E233333rF)rrrcH|d|i}n||d}||dS)a Update tracing_count only. No status updated request_count --> new value tracing_count --> new value if given :param request_count: number of requests already traced :param tracing_count: number of requests left to trace Nr)rrr)rArrrws rDupdate_counts_onlyzAPIClient.update_counts_onlysB  #]3DD%2%244D $rFc6|d|ddS)rr)rfrNrrs rDrzAPIClient.stop s5 Y,133 4 4 4 4 4rFc4|ddidS)rrfrNrrLs rDrzAPIClient.complete*s# 8[122222rFc.|dS)z5 Share tracing task :return: N)rrLs rDsharezAPIClient.share2s rFc.|dSr)rrLs rDrzAPIClient.delete9s rFr)rJrrrrrorKr]rvrrrrr"r rrrrWrr rrr$rrrs@rDrrs7s777X7 ''''''' 8d 8t 8 8 8 8(d(t(((('$''''4D"4444444s4t444415   3 *- 9=    4#4$44443333rFrc>eZdZdZedefdZdeddfdZdS) SendClientz) X-Ray requests API client class rPc&tdtdS)) Base endpoint: requests r6z/api/xray/requestsrrLs rDrKzSendClient.main_endpointFs ::J::::rFrwNcr|jd|}|||ddS)zS Send given data to ClickHouse :param data: dict with data rF)r{NrrArwrys rD__call__zSendClient.__call__MsD (AA4+=+=+?+?AA 8TE 22222rF) rJrrrrrorKrvr,rrFrDr'r'Asg;s;;;X; 3T3d333333rFr'cteZdZdZedefdZdeffd ZdefdZ dedefdZ dede defd Z xZ S) UIAPIClientz, X-Ray User plugin API client class rPc&tdtdS)r)r6z /api/xrayrrLs rDrKzUIAPIClient.main_endpoint[s 11J1111rFc rd|D}tjdi|S)z Construct query string :return: string including system_id and given kwargs, filtered by non-empty values ci|] \}}||| Srrr s rDrz-UIAPIClient._query_string..hsIIIA1=1===rFr)rrr])rAr[ filter_emptyrIs rDr]zUIAPIClient._query_stringbs> JIIII $uww$44|444rFc|t}|jd|}||}|dD]}|ddkrt j|dd}t|5}|d d kr|d |j z |d <|j |d <dddn #1swxYwY|S)zd Get list of tasks and return not processed (full) response from API server )rz/tasks?rqrfrr*r7) digest_sizertimerrrN) r]r!rKrhashlibblake2bencode hexdigestrprocessed_requests)rAqsryr^rfake_idstorages rD get_task_listzUIAPIClient.get_task_listksW   %7%9%9  : :(5555>>(++X& C CDH~,,od+<&=&D&D&F&F246666?ikk #G,, C %//,0,AGD^,^D)(/(B_% C C C C C C C C C C C C C C Cs**C  C$ 'C$ r9cp||}|jd|}||S)zG Get list of requests collected for given tracing task r)z /requests?r]rKr)rAr9r;rys rDget_request_listzUIAPIClient.get_request_listsB    8 8(88B88~~h'''rF request_idcr|||}|jd|}||S)zU Get collected statistics for given request ID of given tracing task )r*rBz /request?r@)rAr9rBr;rys rDget_request_datazUIAPIClient.get_request_datasK  +5 77(77277~~h'''rF)rJrrrrrorKr]rvr>rArrDrrs@rDr.r.Vs2s222X2 5555555t,(((((((((((((((((rFr.ceZdZdZfdZdededefdZede fdZ ede fdZ de fd Z dedd fd Zdd e d edefdZdedd fdZdeefdZddededefdZdefdZdedefdZdedefdZdedefdZxZS)SmartAdviceAPIClient( X-Ray Adviser API client class cLtddSN not_needed)r+rrErArIs rDrEzSmartAdviceAPIClient.__init__$ <00000rFrwschemarPc ||S#t$rf}|jd||jdp |jd}t tdt|zd}~wwxYw)z&Validate given data using given schemaz#Failed to validate API response: %szMalformed API response: %sN) validater r<rerrorsautosrrSro)rArwrNrmsgs rD _validatezSmartAdviceAPIClient._validates K??4(( ( K K K K  CT J J J(2,-!'"+Cq!=>>SIJJ J Ks BA!BBcdtdS)r)https://z/apirrLs rDrKz"SmartAdviceAPIClient.main_endpoint 3,2222rFcdS)z3 Limit fields available for update )rfsourcerkrrLs rDfields_allowedz#SmartAdviceAPIClient.fields_alloweds .-rFc dtD}t}|r|d|f|d|Dt j|dS)za Construct query string :return: string including types and given kwargs cg|]}d|fSrr)r_ts rDrz6SmartAdviceAPIClient._query_string..sAAABFB<AAArFusernamec g|] \}}|||f Srrr s rDrz6SmartAdviceAPIClient._query_string..s%???41aQ?A???rF,)safe)supported_advice_typesr!appendextendrrXrYrZ)rAr[r\ user_contexts rDr]z"SmartAdviceAPIClient._query_strings BA*@AAA)++  7 NNJ 5 6 6 6??6<<>>???@@@|%%gC%888rFNcL|jd}|||dddS)z] Send given data to Adviser microservice :param data: dict with data z /requests/addFr{r_NrKrr+s rDr,zSmartAdviceAPIClient.__call__sC (777 8TE%*  , , , , ,rFryrzc |jd| |j|||j}n8#t $r+}|||dtd|d}~wwxYw||dS)z Perform PATCH request to given endpoint. Add payload as JSON. :param endpoint: target URL :param payload: dict with data to PATCH :return: full response zSending PATCH request to %s)rnrz PATCH failedz/Failed to PATCH data to Smart Advice API serverNFrt) r<rlr>patchrUr rrSrr)rAryrzrrs rD_patchzSmartAdviceAPIClient._patchs 6! # # # i<%%h+2.2m&==DD  i i i++Ax,-._,`,`bbgh i i(((GGGs"A A5 &A00A5cJ|jd}|||ddS)z9 Send statistics to Adviser microservice z/requests/metadataFrtNrjr+s rD send_statzSmartAdviceAPIClient.send_stats3(<<< 8TU ;;;;;rF advice_listcg}t|}|D]g}t|dd}t|j|ddr)t |js||h|S)z Loop over advices and remove those which have non-existing users and those which are invisible. :param advice_list: list of advices received from API advicermetadatar`)r%rr module_namerre)rArpvisible_advicesfilteredradvice_instances rD_filter_advice_listz(SmartAdviceAPIClient._filter_advice_lists*;77 - -D1$x.2HIIO!/"="&z"2:">@@ -5o6QRR - &&t,,,rFTFrvshow_allc|jd||}||d}||t}|r||}|S)z Get list of advice :param filtered: Automatically removes invisible advices and those which are inked to non-existing users. z /advice/list?)ryFrtrwrN)rKr]rrUrrx)rArvryryr^s rDrpz SmartAdviceAPIClient.advice_lists (^^t7I7IS[7I7\7\^^>>(e>DD>>%"''  ://99HrFc|jd|}||d}||tS)z> Get urls/advices information per user`s site z/advice/site_info/Frtr{)rKrrUr)rAr`ryr^s rD site_infozSmartAdviceAPIClient.site_infosP(FFHFF>>(e>DD~~)++ +rF advice_idc|jd|d}||d}||tS)z= Get details of an advice by given advice_id z /v2/advice/z/detailsFrtr{)rKrrUr)rAr~ryr^s rDadvice_detailsz#SmartAdviceAPIClient.advice_detailssS(HHYHHH>>(e>DD~~)++ +rFc fd|D}jd|}||S)z Partial update of an advice by given advice_id. Fields allowed for update are limited by fields_allowed property c.i|]\}}|jv||Sr)r\rs rDrz6SmartAdviceAPIClient.update_advice..!s6***AT(((1(((rFz/advice/)rrKrm)rAr~r[rwrys` rD update_advicez"SmartAdviceAPIClient.update_advices\ *******(==)=={{8T***rFcH|jd}|||ddS)z: Sends analytics data to the microservice z/analytics/eventsF)r_r|rjr+s rDreportzSmartAdviceAPIClient.report&s/(;;;zz(DEuzUUUrFr)TF)rJrrrrErr rUrrorKrr\r]rvr,rmrorrxrrpr}rrrrrrs@rDrFrFs-11111KcK6KcKKKK3s333X3 ....X. 9 9 9 9 9,T,d,,,,HHsHTHSHHHH*>>zz(\:$>#(,133 3rFcl|jd||}||S)Nz /cdn/usage?)rrrs rD get_usagezAWPProvisionAPIClient.get_usagegs<(``T5G5GS]5G5^5^``yy"""rFr)rJrrrrEror]rrKrrrvrxrrrrrrrrs@rDrr.sv11111.....3s333X359NN(N-1N=ANNNN ,c ,3 , , , , ,5#5s5S55553#3s3S33333x}3333 #C########rFr)Irrr6pwdrnr: urllib.parserXr functoolsrtypingrrrrequestsrrrequests.adaptersr requests.exceptionsr $requests.packages.urllib3.util.retryr rNr r !xray.internal.clwpos_safe_importsrradviser.advice_typesrrdrxrayrrSxray.apiclient.schemasrrrclcommon.clwpos_librclcommon.cpapirinternal.constantsrrrinternal.exceptionsrrrinternal.local_countersrinternal.typesr internal.user_plugin_utilsr!r"internal.utilsr$xray.adviser.advice_helpersr%r'rrrrr'r.rFrrrFrDrs   &&&&&&&&&&&&&&&&&&))))))000000666666&&&&&&&& \[[[[[[[ +*****""""""FFFFFFFFFFWWWWWWWWWW888888!!!!!!,+++++??????M?M?M?M?M?M?M?M?`!!!!!!!!033333&)3338q#q#q#q#q#q#q#q#hRRRRR RRRj33333333*:(:(:(:(:(&:(:(:(zXVXVXVXVXV6XVXVXVv;#;#;#;#;#F;#;#;#;#;#rF