3 l d dl Z d dlZd dlZd dlmZmZ d dlZd dlmZmZmZm Z m
Z
m Z d dl m
Z
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d lmZ d d
lmZ d d l m Z m Z d d l m Z m!Z! d d
l"m#Z# dZ$g dZ%g dZ& e Z' G d de' Z( G d de' Z) G d de* Z+ G d de, Z- G d de Z.dS ) N)datetime timedelta)ColumnFloatIntegerStringfuncinsert)
DatabaseErrorSQLAlchemyError)declarative_base) sessionmaker)LveStatsPluginLveStatsPluginTerminated) uidconverter) gm_datetime_to_unixtimestamp)
get_chunks)fix_lost_keep_alivevalidate_database)history history_x60) history_govz /var/lve/v1_migration_last.ts)idmem mem_limit mem_faultmemphylmemphy memphy_faultmep mep_limit mep_faultnproclnproc nproc_faultiopsliops)usernamesum_cpu sum_writesum_readlimit_cpu_on_period_endlimit_read_on_period_endlimit_write_on_period_endcause_of_restrictc N e Zd ZdZdZ eded Z ed ed d Z ede
Z ed e
Z ed
e
Z
ed e Z ed e Z ed
e Z ede Z ed ed d Z ede ZdS ) V1HistoryGovz*
Mapping out v1 gov history table
r tsT primary_keyr( @ r) r* r+ r, r- r. r/ server_id
weightN)__name__
__module__ __qualname____doc__
__tablename__r r r2 r r( r r) r* r+ r, r- r. r/ r6 r8 /builddir/build/BUILDROOT/lve-stats-4.3.2-1.el9.x86_64/opt/cloudlinux/venv/lib/python3.11/site-packages/lvestats/plugins/other/v1_db_migrator.pyr1 r1 < s "M g4 0 0 0Bvj&&**$???HfY&&G{E**Ivj%((H $f%> H H %v&@' J J &'BG L L 2G<<{FF2JJDAAAI
VHg
&
&FFFr? r1 c e Zd ZdZdZ eded Z ede Z ede Z ede Z
ed e Z ed
e Z ed e Z
ed e Z ed
e Z ede Z ede Z ede Z ede Z ede Z ede Z ede Z edej j d Z ede Z ed e d Z ede Z ede Z ede Z ed e Z! ed e Z" ed e Z# ed e Z$ ed e Z% ed!e Z& ed"e Z' ed#e Z(d$S )% V1Historyz&
Mapping out v1 history table
r r Tr3 cpu cpu_limitcpu_maxncpur r! mep_maxioio_maxio_limitr r mem_maxr r" createdr8 r6 r7 r r
memphy_maxr r$ r# nproc_maxr% r& iops_maxr' N))r9 r: r; r< r= r r r rC rD rE rF r r! rG rH rI rJ r r rK r r"
sqlalchemytypesDateTimerL r8 r r6 r r rM r r$ r# rN r% r& rO r' r> r? r@ rB rB P s5 M g4 0 0 0B
&
C{G,,IfY((G 6&' " "D
&
C{G,,IfY((G g B
VHg
&
&Fvj'**H
&
C{G,,IfY((G{G,,I{G,,IfY
0 9tLLLG
VHg
&
&F{FF2JJ//IfY((G
VHg
&
&F g..J6.'22L
VHg
&
&F F7G $ $E{G,,I& 00K 6&' " "Dvj'**H F7G $ $EEEr? rB c e Zd ZdZedfdZddZed Zd Z ddZ
dd Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zd ZdS )V1TimeIntervala
The way it would work - on first run, the /var/lve/v1_migration_last.ts will be non-existant, and we will
use latest timestamp from V1 db as the 'starting point'
After that on each call of get_data we will use that 'starting point' to get_period from start point to 1 hour
before. As soon as our start point is > 30 days old -- we will return as part of get_period third parameter true
which means that ok, the rest of data is too old, lets move on.
V1DBMigrator will convert data for that period, and then will call save_state(from) -- this will be new starting
point for the next plugin run. We will store it in a property (last_ts), and save it to the file.
So, that even if software restarted, we don't just ignore it.
localhostc t || _ || _ d | _ d| _ || _ | d S )N)ts_filer6 last_tslast_uid v1session
read_state)selfr[ rX r6 s r@ __init__zV1TimeInterval.__init__ s; "
" r? Nc p t | j dd 5 }| | | || _ |.| dt
| z |pd| _ | d d d d S # 1 swxY w Y d S )Nwutf-8encoding
rW ) openrX writestrftime
get_ts_formatrY strrZ close)r] r2 uidfs r@ save_ts_to_filez V1TimeInterval.save_ts_to_file s
$,g
6
6
6 !
GGBKK 2 2 4 455 6 6 6 DLs3xx((( # r
GGIII
s BB++B/2B/c dS )Nz%Y-%m-%d %H:%M:%S.%fr> r> r? r@ rh z V1TimeInterval.get_ts_format s %%r? c 0 | | d S N) _save_state)r] r2 s r@ save_timestampz V1TimeInterval.save_timestamp s r? c < | | j | d S rp )rq rY )r] rk s r@ save_uidzV1TimeInterval.save_uid s s+++++r? c | || d S # t $ r@}t j d dt | Y d }~d S d }~ww xY w)N plugin.V1DBMigrator.TimeIntervalz!Unable to save v1 migration TS %s)rm IOErrorlogging getLoggererrorri )r] r2 rk es r@ rq zV1TimeInterval._save_state s u S ) ) ) ) ) u u u @ A A G GHkmpqrmsms t t t t t t t t t us
A$5A A$c " d } t | j dd 5 }t j | | }t | pd }||fcd d d S # 1 swxY w Y d S # t $ r |dfcY S t $ r<}t j d d| j | |dfcY d }~S d }~ww xY w)Nrra rb rW rv zUnable to read %s (%s))
re rX r strptimereadlinerstriprh intrw
ValueErrorrx ry warning)r] r2 rl rk r{ s r@ _read_statezV1TimeInterval._read_state sg
dlC':::
a &qzz||':':'<'P>P>R>RSS!**,,--//52663w
r6MMM @ A A I I(
r6MMMMMM
sG B9 BB, B9 ,B00B9 3B04B9 9D D1D D Dc | \ | _ | _ | j | j t j t j t j
| j
k }|d pt ddd }|t d z | _ d S d S )Nr microseconds)r rY rZ r[ queryr maxrB rL filterr6 firstr r )r] reslast_ts_from_dbs r@ r\ zV1TimeInterval.read_state s &*&6&6&8&8# dm < $$TXi.?%@%@AA +t~=>>uuww
"!f 9Aq(9(9O*YA-F-F-FFDLLL r? c Z | | j t d z
S )Nr r )r\ rY r r] s r@ _to_tszV1TimeInterval._to_ts s* |iQ 7 7 777r? c v t j t d z
| k S )N )days)r nowr r r s r@
is_too_oldzV1TimeInterval.is_too_old s+ |~~ r 2 2 22T[[]]BBr? c 8 | | j S rp )r\ rZ r s r@ get_uidzV1TimeInterval.get_uid s } r? c d S rp r> )r] r( s r@ convert_username_to_uidz&V1TimeInterval.convert_username_to_uid s r? c | \ }}t | }t | }| j t t j || t j | j k t j
t j
}d |D S )Nc g | ] }|j
S r> )r( .0items r@
z9V1TimeInterval._get_history_gov_users.. s 555$
555r? )
get_periodr r[ r r1 r r2 betweenr6 distinctr( group_by)r] from_tsto_tsfrom_ts_to_ts_
usernames_s r@ _get_history_gov_usersz%V1TimeInterval._get_history_gov_users s **/88-e44 N . .
VLO++Hf==|?UY]Yg?g
h
h
Xl+
,
,
Xl+
,
, 65*5555r? c | \ }}| j t t j || t j | j k t j | j k
t j t j }d |D S )Nc g | ] }|j
S r> )r r s r@ r z4V1TimeInterval._get_history_uids.. s ***D***r? ) r r[ r rB r rL r r6 r rZ r r )r] r r uids_s r@ _get_history_uidsz V1TimeInterval._get_history_uids s ** N + +
V!))'599#t~5 t},
Xil
#
#
Xil
#
# +*E****r? c | }| D ]=}| | }|$|| j k r||vr| | >t | S rp )r r r rZ appendsorted)r] uids_listr( rk s r@ get_uidszV1TimeInterval.get_uids s **,, 3355 & &H..x88C3 #6#63i;O;O %%%i r? c b | }| j t d z
}||fS )zEWe want to go 1 hour at a time, up to 1 month back, starting from nowr )hours)r rY r )r] r r s r@ r zV1TimeInterval.get_period s0
,!3!3!33~ r? rp )r9 r: r; r<
STATE_FILEr^ rm staticmethodrh rr rt rq r r\ r r r r r r r r r> r? r@ rT rT w s? +5 & & \& , , , ,u u u u " G G G8 8 8C C C
6
6
6 + + + ! ! ! r? rT c e Zd ZdS )BreakN)r9 r: r; r> r? r@ r r s Dr? r c e Zd ZdZdZdZdZdZdZdZ dZ
dZ dZ dZ
dZd Zd
Zefd Zd Zd
Zd Zd Zd Zd Zd Zd Zd Zd Z d Z e d Z e d Z e d Z!e d Z"d Z#d Z$d Z%d Z&d Z'dS ) V1DBMigratorz./usr/share/lve-stats/plugins/v1_db_migrator.py F< i % NTrU c t j d | _ i | _ g | _ d| _ d| _ | j d | j dz | _ d| _
d | _ d| _ d S )Nzplugin.V1DBMigratorr r zV1 Migration Startedg ?TF)
rx ry log_username_to_uid_cache_no_such_uid_cache_procsr infotimeout _time_commit control_time_conn_database_does_not_existr s r@ r^ zV1DBMigrator.__init__ st $%:;;&(#"$
,--- L3 .
(-%%%r? c , | dd | _ | dd | _ | d | _ | dd dv | _ | d S )N v1_server_idrU r6 v1_connect_stringdebugF)tytrueyesr )getr v2_server_idr lowerr
init_v1_db)r] configs r@
set_configzV1DBMigrator.set_config s "JJ~{ C C "JJ{K @ @!',?!@!@ ZZ--33559UU
r? c | j d| _ d S d}| j | rbt j | j t
| d s)| j d| j d| _ d S t j
| j | j }nF# t $ r9}| j t | d| _ Y d }~d S d }~ww xY w|j |t" j s4| j dt" j | j d| _ d S t' |dt( }|d s|d r#| j d d| _ d S t+ |
| _ t/ | || j | _ | j | j _ d S ) NTz
sqlite:///z Database "%s" does not exist)echoz*Table "%s" in database "%s" does not exist) hide_loggingbase column_error table_errorz)V1 database malformed, migration skipped.bind) r r
startswithospathexistslenr r rP engine
create_enginer r ri dialect has_tablerB r= r V1Baser V1SessionrT get_v1_sessionr
time_intervalr )r] r2 sqlite v1_db_enginer{ results r@ r zV1DBMigrator.init_v1_db s ! ),0D ) F ! , ,V 4 4 RW^^DLbcfgmcncncocoLp=q=q H ;T=S T T T,0D ) F %,::4;QX\Xb:ccLL H SVV $ $ $,0D ) FFFFF #--lI | dd | _ | j rd S | j s| j rD| j d | | d S |
d S )Nprocsr zV1 Migration Done) r r is_doner r r r r cleanupfix_lost_keep_alive_records convert_all)r] lve_datas r@ executezV1DBMigrator.executeE s ll7A.. < F ( D,>,I,I,K,K H 0 1 1 1 LLNNN , , . . . . . r? c t | j }t || j | j | d S )Nr )r6 log_)r r r r r rj )r] sessions r@ r z(V1DBMigrator.fix_lost_keep_alive_recordsP sG 0,DK00022Gt/@txPPPP
r? c d| _ t j t j t j t j dz nQ# t
t f$ r=}| j dt j t | Y d}~nd}~ww xY w t | j } | t t j | j k | dS # t& $ r | Y dS w xY w)z
There is not much to do on clean up. Lets just set flag done = True, and remove plugin
so that on next restart it would't be running any more
:return:
TczUnable to remove %s: %sNr )r r remover PLUGIN_LOCATIONrw OSErrorr rz ri r r r r r r6 r deletecommitr rollback)r] r{ r s r@ r zV1DBMigrator.cleanupU s> \ Il2 3 3 3 Il2S8 9 9 9 9! \ \ \ HNN4l6RTWXYTZTZ [ [ [ [ [ [ [ [ \0,DK00022 MM+ & & - -k.CtGX.X Y Y ` ` b b b NN s* ?A B3BB8A&D EEc b t | }t | }| t t j || t j |k t j | j k
S rp ) r r r r1 r r2 r r( r6 r all)r] r r r( r r s r@ get_v1_gov_dataz V1DBMigrator.get_v1_gov_datai s /88-e44 ! !
U<
V ''&99 %1 &$*;;
SUU
r? c ` | t t j || t j | j k t j |k t j
S rp ) r r rB r rL r r6 r r order_byr )r] r r rk s r@ get_v1_datazV1DBMigrator.get_v1_dataw sz ! !
U9
V!))'599#t'88 #
Xil
#
#
SUU
r? c | | } g }| ||| D ]2}| || j }| | 3|r?t | D ]/} | j t t | 0g }
|rJ|dk rD|
||| D ],}| | } |
| -|
r?t |
D ]1} | j t t | 0d S d S # t t f$ rP} | | j dt% | | j s| Y d } ~ d S d } ~ wt( $ rB} | | j d t/ | d } ~ ww xY w)Nrootz!Can not save data to database: %szPlugin is terminated.)convert_uid_to_usernamer convert_rowr extendr r r r
r r convert_gov_rowr r r r r r ri
skip_on_errorr r r r )
r] r r rk transr( v2_rows_insert_listrowv2_rowschunkv2_gov_rows_insert_list v2_gov_rowsr{ s
r@
_convert_datazV1DBMigrator._convert_data s9 //44 !"$ ''<<
4
4 **3 <<#**73333"
? '(; < < ? ?EJ&&vg>>>>&( #
@H . . //II @ @C"&"6"6s";";K+22;????&
C '(? @ @ C CEJ&&vk':':EBBBB
C
CC C / NN H @#a&& I I I%
' ! ! ! LLNNN HNN2 3 3 3''q !s% D D7 7G"AF
G" =G G"c 8 t j | j z
S rp )timer r s r@
_work_timezV1DBMigrator._work_time s y{{TX%%r? c R | j | z
| j dz k S )Ng333333?)r r r r s r@ _need_breakzV1DBMigrator._need_break s& |doo / //$2Cc2IIIr? c ^ | j d|| | j }|sd S | j }|D ]}| |||| | j | | j d|||| | j rO|
r;| j d| t |j r|
d S d S )Nz Start converting from %s to %sz4Converted from %s to %s uid: %s; plugin work time %sz$Stop converting; plugin work time %s)r r r r r beginr rt r r r r is_activer )r] r r uidsr rk s r@ convert_datazV1DBMigrator.convert_data s9 7%HHH!**,, F
"" C wsE : : : ' ' , , , HNNF!!
T%5%5%7%7
:OO%% gg
? LLNNNNN r? c | j 5 | _ | s| j sy| j \ }}| || | j | | s| j yn# t $ r Y nw xY wt j
}d d d n # 1 swxY w Y t j
|z
}t | j | | _ | j
d| d S )NzCommit time %s)r r r r r r r r" rr r r r r r r )r] r r
time_start commit_times r@ r zV1DBMigrator.convert_all s|
[
%DJ
**,, ?T5G5R5R5T5T ?%)%7%B%B%D%DNGU%%gu555&55g>>> **,, ?T5G5R5R5T5T ?
J % % % % % % % % % % % % % % % ikkJ. 1; ? ? '55555s5 C7B&CC7
CC7CC77C;>C;c | |k rdS dS )Nr r r> )limit_maxs r@ fault_countzV1DBMigrator.fault_count s D==11r? c V t | j | j |d< d S )N
iops_fault)r r) r' rO v1_rowv2_rows r@ convert_iops_faultsz V1DBMigrator.convert_iops_faults s( , 7 7 fo V V| r? c | j dz |d< | j dz |d< t | j | j |d< d S )Ni rH rJ io_fault)rH rJ r r) rI r, s r@
convert_iozV1DBMigrator.convert_io sI y4't #_t 3z ) 5 5fov} U Uzr? c t d|z | z |dz dz }t ||| z dz }t |d|z | z }|||fS )a
v1 holds CPU relative to total cores, where on 4 core system 1 core is 25%
it also limits by ncpu (whatever is less), so on 4 cores system 2 ncpu and 30% is 30%
of all cores (as 2ncpu = 50%, and we take smaller), and 2 ncpu and 70% is 50%, as
2ncpu = 50% / we take smaller
To switch to new limit, we need to talke old limit and multiply it by 100
So 25% on 4 core system in v1 (1 core), is 25 * 4 * 100 = 10,000
d )minr r) )r rC rD rE rF v2_cpu_limitv2_cpu
v2_cpu_faultss r@ convert_cpu_zV1DBMigrator.convert_cpu_ sj 3?U2D3J4DEE \3;#455 %00 sW}u?TUU
|]22r? c z | ||j |j |j |j \ |d< |d< |d< d S )NrC rD cpu_fault)r9 rC rD rE rF )r] r r. r s r@ convert_cpuzV1DBMigrator.convert_cpu sG BFBSBS 37CM3;C
C
?u
vk*F;,?,?,?r? c || j v r
| j | S t j | }|| j |< || j d| |S )Nz Can not find uid for user %s)r r username_to_uid_localr r )r] r( rk s r@ r z$V1DBMigrator.convert_username_to_uid s^ t2 2 2.x8 80::03#H- ; H ;X F F F
r? c || j v rd S | j D ]\ }}||k r|c S t j | }|6| j | | j d| n
|| j |< |S )Nz!Can not find user name for uid %s)r r itemsr uid_to_username_localr r r )r] rk username_uid_s r@ r z$V1DBMigrator.convert_uid_to_username s $) ) ) F # : @ @ B B ! !OItd{{ 6s;; # * *3 / / / H @# F F F F58D ' 2r? c * |j }g }t d|j D ]s}| j |d|z z
d}t D ]}t || ||< | | d }|r||d< | | t|S )Nr r )r6 r2 r( rk ) r2 ranger8 r V2_GOV_KEYSgetattrr popr )r] r r r i
v2_gov_rowkeyrk s r@ r z V1DBMigrator.convert_gov_row s q#*%% * *A'+'8QOOJ"
4
4")#s"3"3
3..z~~j/I/IJJC
*$'
5!
j)))
r? c l t |j }g }t d|j D ]}| j |d|z z
d}t
D ]}t
|| ||< | ||| | || | || |
| |S )Nr r )r6 rL ) r rL rE r8 r V2_KEYSrG r< r2 r/ r )r] r r r r rI r. rK s r@ r
zV1DBMigrator.convert_row s ,S[99q#*%% " "A#'#4a PPF
0
0 %c3 / /s S&% 0 0 0 OOC ( ( ( $ $S& 1 1 1 MM& ! ! ! !
r? )(r9 r: r; r r r periodorderr r r r r r r r^ r r r r r r r r r r r r r" r r r) r/ r2 r9 r< r r r r
r> r? r@ r r s FOGG
F E IM EM L L
.
.
. ' $R $R $R $RL
(
! ! !:& & &J J J 6 6 6 6 \ W W \W V V \V 3 3 \3"
r? r )/rx r r r r rP r r r r r r
sqlalchemy.excr r sqlalchemy.ext.declarativer
sqlalchemy.ormr lvestats.core.pluginr r lvestats.libr lvestats.lib.commons.dateutilr lvestats.lib.commons.funcr lvestats.lib.dbenginer r lvestats.orm.historyr r lvestats.orm.history_govr r rM rF r r1 rB objectrT Exceptionr r r> r? r@ r\ si ( ( ( ( ( ( ( ( C C C C C C C C C C C C C C C C 9 9 9 9 9 9 9 9 7 7 7 7 7 7 ' ' ' ' ' ' I I I I I I I I % % % % % % F F F F F F 0 0 0 0 0 0 H H H H H H H H 5 5 5 5 5 5 5 5 0 0 0 0 0 0
,
$
' ' ' ' '6 ' ' '($% $% $% $% $% $% $% $%N} } } } } V } } } @ I n n n n n> n n n n nr?