!(U(
d dl mZ d dlmZmZ ddlmZmZ ddlm Z m
Z
m Z m Z m
Z
mZmZmZmZmZmZmZmZmZmZmZ G d d Z G d d e Z G d
d e Z G d d
e Z G d de Z G d de Z G d de Z G d de Z G d de Z! ed dee" dee" d e#fd Z$ ed d&d!e"d"e%d#e#d e%fd$ Z&d%S )' ) lru_cache)ListOptional ) COMMON_SAFE_ASCII_CHARACTERS UNICODE_SECONDARY_RANGE_KEYWORD)is_accentuatedis_asciiis_case_variableis_cjk is_emoticon is_hangul is_hiragana is_katakanais_latinis_punctuation is_separator is_symbolis_thaiis_unprintable
remove_accent
unicode_rangec V e Zd ZdZdedefdZdeddfdZd dZe de
fd Z dS )
MessDetectorPluginzy
Base abstract class used for mess detection plugins.
All detectors MUST extend and implement given methods.
characterreturnc t )z@
Determine if given character should be fed in.
NotImplementedErrorselfr s g/builddir/build/BUILD/cloudlinux-venv-1.0.10/venv/lib/python3.11/site-packages/charset_normalizer/md.pyeligiblezMessDetectorPlugin.eligible
"! Nc t )z
The main routine to be executed upon character.
Insert the logic in witch the text would be considered chaotic.
r r s r" feedzMessDetectorPlugin.feed% s
"!r% c t )zB
Permit to reset the plugin to the initial state.
r r! s r" resetzMessDetectorPlugin.reset, r$ r% c t )z
Compute the chaos ratio based on what your feed() has seen.
Must NOT be lower than 0.; No restriction gt 0.
r r) s r" ratiozMessDetectorPlugin.ratio2 s
"!r% r N) __name__
__module__ __qualname____doc__strboolr# r' r* propertyfloatr, r% r" r r s
"# "$ " " " " "c "d " " " "" " " " "u " " " X" " "r% r c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
TooManySymbolOrPunctuationPluginr Nc L d| _ d| _ d| _ d | _ d| _ d S )Nr F)_punctuation_count
_symbol_count_character_count_last_printable_char_frenzy_symbol_in_wordr) s r" __init__z)TooManySymbolOrPunctuationPlugin.__init__< s0 '( "#%& 37!,1###r% r c * | S N isprintabler s r" r# z)TooManySymbolOrPunctuationPlugin.eligibleD $$&&&r% c ( | xj dz
c_ || j k ro|t vrft | r| xj dz
c_ nF| du r0t
| r!t | du r| xj dz
c_ || _ d S )Nr F ) r< r= r r r: isdigitr r
r; r s r" r' z%TooManySymbolOrPunctuationPlugin.feedG s "
2 2 2!=== i((
(''1,'''!!##u,, i(( - **e33""a'""$-!!!r% c 0 d| _ d| _ d| _ d S Nr )r: r< r; r) s r" r* z&TooManySymbolOrPunctuationPlugin.resetY s "# ! r% c ^ | j dk rdS | j | j z | j z }|dk r|ndS )Nr g333333?)r< r: r; )r! ratio_of_punctuations r" r, z&TooManySymbolOrPunctuationPlugin.ratio^ sK A % %3
#d&8 8 !'" (AV'V (=(E(E$$3Nr% r- rM r6 r% r" rO rO j s ) ) ) )## #$ # # # #)c )d ) ) ) ) $ $ $ $ Ou O O O XO O Or% rO c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
UnprintablePluginr Nc " d| _ d| _ d S rI )_unprintable_countr< r) s r" r? zUnprintablePlugin.__init__ s '( %& r% r c dS NTr6 r s r" r# zUnprintablePlugin.eligible tr% c d t | r| xj dz
c_ | xj dz
c_ d S rV )r r] r< r s r" r' zUnprintablePlugin.feed s@ ) $ $ ) # #q ( # # " r% c d| _ d S rI )r] r) s r" r* zUnprintablePlugin.reset s "# r% c @ | j dk rdS | j dz | j z S )Nr rK )r< r] r) s r" r, zUnprintablePlugin.ratio s+ A % %3'!+t/DDDr% r- rM r6 r% r" r[ r[ s ' ' ' '# $ #c #d # # # #
$ $ $ $ Eu E E E XE E Er% r[ c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
SuspiciousDuplicateAccentPluginr Nc 0 d| _ d| _ d | _ d S rI _successive_countr< _last_latin_characterr) s r" r? z(SuspiciousDuplicateAccentPlugin.__init__ s &' %& 48"""r% r c H | ot | S rA )rT r r s r" r# z(SuspiciousDuplicateAccentPlugin.eligible s! "":x ':'::r% c l | xj dz
c_ | j t | rt | j rr| r)| j r| xj dz
c_ t | t | j k r| xj dz
c_ || _ d S rV )r< rj r isupperri r r s r" r' z$SuspiciousDuplicateAccentPlugin.feed s " & 2 y))
3 t 9::
3 ""
,t'A'I'I'K'K
,&&!+&& Y''=9S+T+TTT&&!+&&%."""r% c 0 d| _ d| _ d | _ d S rI rh r) s r" r* z%SuspiciousDuplicateAccentPlugin.reset s !" ! %)"""r% c @ | j dk rdS | j dz | j z S )Nr rK rF )r< ri r) s r" r, z%SuspiciousDuplicateAccentPlugin.ratio s+ A % %3&*d.CCCr% r- rM r6 r% r" rf rf s 9 9 9 9 ;# ;$ ; ; ; ; /c /d / / / / * * * *
Du D D D XD D Dr% rf c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
SuspiciousRanger Nc 0 d| _ d| _ d | _ d S rI )"_suspicious_successive_range_countr< _last_printable_seenr) s r" r? zSuspiciousRange.__init__ s 78/%& 37!!!r% r c * | S rA rB r s r" r# zSuspiciousRange.eligible rD r% c D | xj dz
c_ | st | s |t v r d | _ d S | j || _ d S t | j }t | }t
|| r| xj dz
c_ || _ d S rV )r< isspacer r rt r is_suspiciously_successive_rangers )r! r unicode_range_aunicode_range_bs r" r' zSuspiciousRange.feed s "
i(( 888(,D % F $ ,(1D % F)6t7P)Q)Q)6y)A)A +O_ M M 9 3 3q 8 3 3$-!!!r% c 0 d| _ d| _ d | _ d S rI )r< rs rt r) s r" r* zSuspiciousRange.reset s ! 23/$(!!!r% c T | j dk rdS | j dz | j z }|dk rdS |S )Nr rK rF g?)r< rs )r! ratio_of_suspicious_range_usages r" r, zSuspiciousRange.ratio sH A % %3
3a 7 !2"' +S 0 03..r% r- rM r6 r% r" rq rq s 8 8 8 8
'# '$ ' ' ' '.c .d . . . ..) ) ) )
/u / / / X / / /r% rq c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
SuperWeirdWordPluginr Nc d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d S )Nr F ) _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr< _bad_character_count_buffer_buffer_accent_countr) s r" r? z SuperWeirdWordPlugin.__init__ sO !$% () */!). %& )*! )*!!!r% r c dS r_ r6 r s r" r# z SuperWeirdWordPlugin.eligible r` r% c | r| xj |z
c_ t | r| xj dz
c_ | j du r|t | du st | r\t
| du rKt | du r:t | du r)t | du rt | du rd| _ d S | j sd S | s t | st | r"| j r| xj dz
c_ t | j }| xj |z
c_ |dk re| j |z dk rd| _ t | j d r6| j d r| xj dz
c_ d| _ |dk r | j r| xj dz
c_ d| _ | j r9| xj dz
c_ | xj t | j z
c_ d| _ d| _ d| _ d | _ d S |d
vr>| du r*t/ | r d| _ | xj |z
c_ d S d S d S d S ) Nr FT g(\? r r > _-<=>|~)rT r r r r r r r r r r rw r r r lenr< r rm r r r rG r )r! r
buffer_lengths r" r' zSuperWeirdWordPlugin.feed s LLI %LL i((
/))Q.))(E11 i((E11^I5N5N19%%.. i((E11 **e33 **e33I&&%//+/( F| F " )#<#<" &@LY@W@W" &l" &
! !$T\!2!2M ! !] 2 ! ! !!,}->>)),1)',D $ DL()D % % % @ @ @!!##u,,)$$ - )-D % LLI %LLLL
A @,,,,r% c v d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d S )Nr Fr )r r r r r r< r r r) s r" r* zSuperWeirdWordPlugin.reset= sG $)!#( ! $%!#$ r% c P | j dk r
| j dk rdS | j | j z S )N
r rK )r r r r< r) s r" r, zSuperWeirdWordPlugin.ratioG s3 r ! !d&>!&C&C3(4+@@@r% r- rM r6 r% r" r r s + + + + # $ 4&c 4&d 4& 4& 4& 4&l% % % % Au A A A XA A Ar% r c ^ e Zd ZdZd
dZdedefdZdeddfdZd
dZ e
de fd Z dS ) CjkInvalidStopPluginu
GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
can be easily detected. Searching for the overuse of '丅' and '丄'.
r Nc " d| _ d| _ d S rI _wrong_stop_count_cjk_character_countr) s r" r? z CjkInvalidStopPlugin.__init__U s &' )*!!!r% r c dS r_ r6 r s r" r# z CjkInvalidStopPlugin.eligibleY r` r% c t |dv r| xj dz
c_ d S t | r| xj dz
c_ d S d S )N> 丄 丅r )r r r r s r" r' zCjkInvalidStopPlugin.feed\ sZ & & " "a ' " " F ) + % % * % % % % + +r% c " d| _ d| _ d S rI r r) s r" r* zCjkInvalidStopPlugin.resetc s !" $%!!!r% c : | j dk rdS | j | j z S )N rK )r r r) s r" r, zCjkInvalidStopPlugin.ratiog s& $r ) )3%(AAAr% r- )
r. r/ r0 r1 r? r2 r3 r# r' r* r4 r5 r, r6 r% r" r r O s
+ + + +# $ +c +d + + + +& & & & Bu B B B XB B Br% r c Z e Zd Zd dZdedefdZdeddfdZd dZe de
fd Z dS )
ArchaicUpperLowerPluginr Nc h d| _ d| _ d| _ d| _ d| _ d | _ d| _ d S )NFr T)_buf _character_count_since_last_sep _successive_upper_lower_count#_successive_upper_lower_count_finalr< _last_alpha_seen_current_ascii_onlyr) s r" r? z ArchaicUpperLowerPlugin.__init__o s? 45,23*890%& /3 )- r% r c dS r_ r6 r s r" r# z ArchaicUpperLowerPlugin.eligible| r` r% c | ot | }|du }|r| j dk rt| j dk r4| du r | j du r| xj | j z
c_ d| _ d| _ d | _ d| _ | xj dz
c_ d| _ d S | j du rt | du rd| _ | j | r| j s-| rB| j r)| j du r| xj dz
c_ d| _ nd| _ nd| _ | xj dz
c_ | xj dz
c_ || _ d S )NFr @ r TrF )
rT r r rG r r r r r r< r
rm islower)r! r is_concerned chunk_seps r" r' z ArchaicUpperLowerPlugin.feed s ((**J/? /J/J E) =AA4:: %%''500,5588688 23D .34D 0$(D ! DI ! !Q & ! !'+D $ F #t + +0C0Cu0L0L',D $ ,!!##
"(=(E(E(G(G
"!!##
"(,(=(E(E(G(G
" 9$$66!;66 %DII $DII ! " ,,1,, ) r% c h d| _ d| _ d| _ d| _ d | _ d| _ d| _ d S )Nr FT)r< r r r r r r r) s r" r* z ArchaicUpperLowerPlugin.reset s? ! /0,-.*340 $ #' r% c : | j dk rdS | j | j z S )Nr rK )r< r r) s r" r, z ArchaicUpperLowerPlugin.ratio s& A % %37$:OOOr% r- rM r6 r% r" r r n s . . . .# $ (*c (*d (* (* (* (*T( ( ( ( Pu P P P XP P Pr% r )maxsizery rz r c | |dS | |k rdS d| v rd|v rdS d| v sd|v rdS d| v sd|v r
d| v sd|v rdS | d | d }}|D ]}|t v r ||v r dS | dv |dv }}|s|r
d | v sd |v rdS |r|rdS d
| v sd
|v rd | v sd |v rdS | d k s|d k rdS d | v s d |v s| d v r|d v rd
| v sd
|v rdS d| v sd|v rdS dS )za
Determine if two Unicode range seen next to each other can be considered as suspicious.
NTFLatin Emoticons Combining )HiraganaKatakanaCJKHangulz Basic Latin)r r PunctuationForms)splitr )ry rz keywords_range_akeywords_range_belrange_a_jp_charsrange_b_jp_charss r" rx rx s /"9t/))u/!!g &@&@uo%%)G)Gu ?""g &@&@&&+ *H*Hu)8)>)> * * S!! '
0 0 0
! ! !55 "
33
' ,
E_$<$<u , u?""h/&A&A O # #u'?'?5 m + +-/O/O5 E_$<$<333 7 7 7 O + +}/O/O5 o % %O)C)C5 4r% i 皙?Fdecoded_sequencemaximum_thresholddebugc d t D }t | dz }d}|dk rd}n |dk rd}nd}t | d z t | D ]m\ }}|D ],} | | r| | -|d
k r ||z d
k s ||dz
k r!t d |D }||k r nn|r |D ] }
t |
j |
j
t |d S )
zw
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
c " g | ] } |
S r6 r6 ).0md_classs r"
z mess_ratio.. s+ + + +
+ + +r% r rK i r r
r c 3 $ K | ] }|j V d S rA )r, )r dts r" z mess_ratio.. s$ !?!?r"(!?!?!?!?!?!?r% ) r __subclasses__r zipranger# r' sumprint __class__r, round) r r r detectorslengthmean_mess_ratio!intermediary_mean_mess_ratio_calcr indexdetectorr s r"
mess_ratior sh + +#5#D#D#F#F+ + +I &''!+F O
||13)) 4 ,.)),/) 04 7v G G 5! ) )H ++
)
i(((
AII%"CCqHH
fqj
!!?!?Y!?!?!? ? ?O "333 * * *B ", ) ) ) ) ! $ $$r% N)r F)' functoolsr typingr r constantr r utilsr r
r r r
r r r r r r r r r r r r r8 rO r[ rf rq r r r r2 r3 rx r5 r r6 r% r" r s ! ! ! ! ! ! ! ! S S S S S S S S ( " " " " " " " "D,L ,L ,L ,L ,L'9 ,L ,L ,L^O O O O O 1 O O O4E E E E E* E E E0"D "D "D "D "D&8 "D "D "DJ1/ 1/ 1/ 1/ 1/( 1/ 1/ 1/hWA WA WA WA WA- WA WA WAt B B B B B- B B B>IP IP IP IP IP 0 IP IP IPX 4C c]C5=c]C C C C CL 4IN'% '%'%.3'%BF'%
'% '% '% '% '% '%r%