VE . d dl mZmZmZ d dlmZmZmZ d dlm Z m
Z
m Z d dl m
Z
mZmZ d dlmZmZmZ G d de Z G d d e Z G d
d e Z G d d
e Z G d de Z G d de Z G d de ZdS ) )EUCTW_CHAR_TO_FREQ_ORDEREUCTW_TABLE_SIZE EUCTW_TYPICAL_DISTRIBUTION_RATIO)EUCKR_CHAR_TO_FREQ_ORDEREUCKR_TABLE_SIZE EUCKR_TYPICAL_DISTRIBUTION_RATIO)GB2312_CHAR_TO_FREQ_ORDERGB2312_TABLE_SIZE!GB2312_TYPICAL_DISTRIBUTION_RATIO)BIG5_CHAR_TO_FREQ_ORDERBIG5_TABLE_SIZE BIG5_TYPICAL_DISTRIBUTION_RATIO)JIS_CHAR_TO_FREQ_ORDERJIS_TABLE_SIZE JIS_TYPICAL_DISTRIBUTION_RATIOc B e Zd ZdZdZdZdZd Zd Zd Z d Z
d Z d
Z d S ) CharDistributionAnalysisi gGz?g{Gz? c d | _ d | _ d | _ d | _ d | _ d | _ | d S N)_char_to_freq_order _table_sizetypical_distribution_ratio_done _total_chars _freq_charsresetselfs /builddir/build/BUILDROOT/alt-python311-pip-21.3.1-4.el9.x86_64/opt/alt/python311/lib/python3.11/site-packages/pip/_vendor/chardet/chardistribution.py__init__z!CharDistributionAnalysis.__init__. sF $(
+/'
c 0 d| _ d| _ d| _ dS )z reset analyser, clear any stateF N)r r r r s r r z CharDistributionAnalysis.reset= s!
r" c |dk r| | }nd}|dk r>| xj dz
c_ || j k r%d| j | k r| xj dz
c_ dS dS dS dS )z"feed a character with known length r$ r i N) get_orderr r r r )r charchar_lenorders r feedz CharDistributionAnalysis.feedF s q==NN4((EEE A:: " t'''1%888$$)$$$$ : ('88r" c | j dk s| j | j k r| j S | j | j k r,| j | j | j z
| j z z }|| j k r|S | j S )z(return confidence based on existing datar$ )r r MINIMUM_DATA_THRESHOLDSURE_NOr SURE_YES)r rs r get_confidencez'CharDistributionAnalysis.get_confidenceT s| ! !T%59T%T%T< 0 0 0!d&7$:J&J2&3 4A4= } r" c " | j | j k S r )r ENOUGH_DATA_THRESHOLDr s r got_enough_dataz(CharDistributionAnalysis.got_enough_datad s 4#===r" c dS )Nr' )r byte_strs r r( z"CharDistributionAnalysis.get_orderi s
rr" N)
__name__
__module__ __qualname__r4 r0 r/ r. r! r r, r2 r5 r( r7 r" r r r ( s HG
* * * > > >
r" r c $ e Zd Z fdZd Z xZS )EUCTWDistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) superr= r! r r r r r r r __class__s r r! z"EUCTWDistributionAnalysis.__init__r >
'..77999#; +*J'''r" c J |d }|dk rd|dz
z |d z dz
S dS )Nr$ ^ r r' r7 r r8
first_chars r r( z#EUCTWDistributionAnalysis.get_orderx ;
a[
d*+hqk9D@ @2r" r9 r: r; r! r(
__classcell__rA s @r r= r= q L K K K K K r" r= c $ e Zd Z fdZd Z xZS )EUCKRDistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) r? rO r! r r r r r r r@ s r r! z"EUCKRDistributionAnalysis.__init__ rB r" c J |d }|dk rd|dz
z |d z dz
S dS )Nr$ rE r rF r' r7 rG s r r( z#EUCKRDistributionAnalysis.get_order rI r" rJ rL s @r rO rO rM r" rO c $ e Zd Z fdZd Z xZS )GB2312DistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) r? rT r! r r r
r r r r@ s r r! z#GB2312DistributionAnalysis.__init__ s>
($//88:::#< ,*K'''r" c Z |d |d }}|dk r|dk rd|dz
z |z dz
S dS )Nr$ r rR rF rE r' r7 r r8 rH second_chars r r( z$GB2312DistributionAnalysis.get_order sI
#+1+x{K
$ [D%8%8d*+k9D@ @2r" rJ rL s @r rT rT sL L L L L L r" rT c $ e Zd Z fdZd Z xZS )Big5DistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) r? rZ r! r r r
r r r r@ s r r! z!Big5DistributionAnalysis.__init__ s>
&--66888#: **I'''r" c | |d |d }}|dk r%|dk rd|dz
z |z dz
dz S d|dz
z |z dz
S dS ) Nr$ r rF ? @ r' r7 rW s r r( z"Big5DistributionAnalysis.get_order si
#+1+x{K
d""j4 /0;>EJJj4 /0;>EE2r" rJ rL s @r rZ rZ sL J J J J J r" rZ c $ e Zd Z fdZd Z xZS )SJISDistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) r? rb r! r r r r r r r@ s r r! z!SJISDistributionAnalysis.__init__ s>
&--66888#9 )*H'''r" c |d |d }}|dk r|dk r d|dz
z }n|dk r|dk r d|dz
dz z }nd S ||z d
z
}|d k rd }|S ) Nr$ r r' r` r7 )r r8 rH rX r+ s r r( z"SJISDistributionAnalysis.get_order s
#+1+x{K
$ Z4%7%7:,-EED zT'9'9:,r12EE2 #d* E r" rJ rL s @r rb rb sL I I I I I r" rb c $ e Zd Z fdZd Z xZS )EUCJPDistributionAnalysisc t t | t | _ t
| _ t | _ d S r ) r? rm r! r r r r r r r@ s r r! z"EUCJPDistributionAnalysis.__init__ s>
'..77999#9 )*H'''r" c J |d }|dk rd|dz
z |d z dz
S dS )Nr$ rE rF r r' r7 )r r8 r) s r r( z#EUCJPDistributionAnalysis.get_order s8
{ 4<<% 3d: :2r" rJ rL s @r rm rm sL I I I I I r" rm N) euctwfreqr r r euckrfreqr r r
gb2312freqr r
r big5freqr r
r jisfreqr r r objectr r= rO rT rZ rb rm r7 r" r rw s> 8: : : : : : : : : :: : : : : : : : : :< < < < < < < < < <8 8 8 8 8 8 8 8 8 86 6 6 6 6 6 6 6 6 6F F F F Fv F F FR 8 & 8 & !9 & 7 , 7 2 8 r"