"
J d dl mZmZ ddlmZ ddlmZ G d de ZdS ) )ListUnion )
CharSetProber) ProbingStatec 8 e Zd ZdZdZdZd fd Zd fd Zede fd Z
ede fd Z de fd
Z
de fd Zdefd Zdefd
ZdefdZdefdZdee ddfdZdee ddfdZdeeef defdZ edefd Z de fdZ xZ S )
UTF1632Proberad
This class simply looks for occurrences of zero bytes, and infers
whether the file is UTF16 or UTF32 (low-endian or big-endian)
For instance, files looking like ( [nonzero] )+
have a good probability to be UTF32BE. Files looking like ( [nonzero] )+
may be guessed to be UTF16BE, and inversely for little-endian varieties.
gGz?returnNc 2 t d| _ dgdz | _ dgdz | _ t
j | _ g d| _ d| _ d| _
d| _ d| _ d| _
d| _ | d S )Nr r r r r F)super__init__position zeros_at_modnonzeros_at_modr DETECTING_statequadinvalid_utf16beinvalid_utf16leinvalid_utf32beinvalid_utf32le'first_half_surrogate_pair_detected_16be'first_half_surrogate_pair_detected_16leresetself __class__s g/builddir/build/BUILD/cloudlinux-venv-1.0.10/venv/lib/python3.11/site-packages/chardet/utf1632prober.pyr zUTF1632Prober.__init__) s
C!G !sQw ", LL $ $ $ $ 7<47<4
c
t d| _ dgdz | _ dgdz | _ t
j | _ d| _ d| _ d| _
d| _ d| _ d| _
g d| _ d S )Nr r
Fr )r r r r r r r r r r r r r r r r s r! r zUTF1632Prober.reset8 s
C!G !sQw ", $ $ $ $ 7<47<4 LL r" c | rdS | rdS | rdS | rdS dS )Nzutf-32bezutf-32lezutf-16bezutf-16lezutf-16)is_likely_utf32beis_likely_utf32leis_likely_utf16beis_likely_utf16ler s r! charset_namezUTF1632Prober.charset_nameF sk ! ! # # : ! ! # # : ! ! # # : ! ! # # :xr" c dS )N r) s r! languagezUTF1632Prober.languageS s rr" c 2 t d| j dz S )N ?g @maxr r) s r! approx_32bit_charsz UTF1632Prober.approx_32bit_charsW 3
+,,,r" c 2 t d| j dz S )Nr0 g @r1 r) s r! approx_16bit_charsz UTF1632Prober.approx_16bit_charsZ r4 r" c | }|| j k ok| j d |z | j k oR| j d |z | j k o9| j d |z | j k o | j d |z | j k o| j S Nr r )r3 MIN_CHARS_FOR_DETECTIONr EXPECTED_RATIOr r r approx_charss r! r% z UTF1632Prober.is_likely_utf32be] s ..00 t ;;
a < /$2E E
)!!$|3d6II
)!!$|3d6II
) $Q',69LL
) ((
r" c | }|| j k ok| j d |z | j k oR| j d |z | j k o9| j d |z | j k o | j d |z | j k o| j S r8 )r3 r; r r< r r r= s r! r& z UTF1632Prober.is_likely_utf32leg s ..00 t ;;
#l 2T5H H
)!!$|3d6II
)!!$|3d6II
) !!$|3d6II
) ((
r" c | }|| j k oU| j d | j d z |z | j k o.| j d | j d z |z | j k o| j S )Nr r: r r9 )r6 r; r r< r r r= s r! r' z UTF1632Prober.is_likely_utf16beq ..00 t ;;
!!
$t';A'>
>, N!
"
)"1%(9!(<< L!"
) ((
r" c | }|| j k oU| j d | j d z |z | j k o.| j d | j d z |z | j k o| j S )Nr r9 r r: )r6 r; r r< r r r= s r! r( z UTF1632Prober.is_likely_utf16le{ rA r" r c H |d dk s:|d dk s.|d dk r)|d dk r d|d cxk rdk r
n nd| _ |d dk s;|d dk s/|d dk r,|d dk r"d|d cxk rdk rn d S d| _ d S d S d S d S )
z
Validate if the quad of bytes is valid UTF-32.
UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
excluding 0x0000D800 - 0x0000DFFF
https://en.wikipedia.org/wiki/UTF-32
r r r9 Tr: N)r r )r r s r! validate_utf32_charactersz'UTF1632Prober.validate_utf32_characters s
GqLLAw~~Q1 aA $$q'2I2I2I2IT2I2I2I2I2I#'D GqLLAw~~Q1 aA $$q'2I2I2I2IT2I2I2I2I2I2I#'D 2I2Ir" pairc | j s>>..ty1~>>>Avv!$'''1,''''$T***a/*** MMQ MMMzr" c | j t j t j hv r| j S | dk rt j | _ n | j dk rt j | _ | j S )Ng?i )r r NOT_MEFOUND_ITget_confidencer r) s r! rO zUTF1632Prober.state sf ;<. 0EF F F; 4 ' '&/DKK
]X
%
% '-DK{r" c | s<| s(| s| rdndS )Ng333333?g )r( r' r& r% r) s r! rV z UTF1632Prober.get_confidence sh &&((
))++
))++
))++
DD
r" )r N) __name__
__module__ __qualname____doc__r; r< r r propertystrr* r. floatr3 r6 boolr% r& r' r( r intrG rL r bytes bytearrayr rR rO rV
__classcell__)r s @r! r r s !N
! ! ! ! ! !
c
X
# X-E - - - --E - - - -
4
4
4
4
(d3i (D ( ( ( (, ,d3i ,D , , , ,@
U5)#3 4
|
X
r" r N)typingr r
charsetproberr enumsr r r- r" r! rg s * ( ( ( ( ( ( F
F
F
F
F
M F
F
F
F
F
r"