VK8 G d Z dZddlmZ ddlmZmZ ddlZddl Z ddl
m Z ddl Z
ddlZddlZddlT ddlmZ ej d ej Z ej d
ej ZddlZej g d z Z[ G d d
ej d
d Zd Zd Zd ZdZ dZ e ede z z ee z Z dZ dZ dZ!dZ"dZ# ee e!e"e# Z$dZ% edd ee% z Z&d e%z Z' ee&e' Z( ed e(d z Z) ee)e(e$ Z*d Z+e j, d! Z- e e+ Z.d"Z/d#Z0d$Z1d%Z2 ee.d&z e.d'z Z3 ee.d(z e.d)z Z4 e e5ej6 e7ed*+ Z8 ed,e8 Z9 ee*e9e4e Z:e e:z Z; ee.d-z ed.d z e.d/z ed0d z Z< ed1e e3 Z=e ee=e*e9ei Z? e+ D ]"Z@e/e?e@d.z < e0e?e@d0z < e1e?e@d&z < e2e?e@d'z < #[@ eA ZB eA ZC e+ D ]FZDeDd0z eDd.z fD ]ZEeBF eE eDd'z eDd&z fD ]ZEeCF eE G[D[Ed2ZG G d3 d4eH ZI G d5 d6eH ZJ G d7 d8 ZKd9 ZLd: ZMd; ZNd< Zd= ZOd> ZPd? ZQd@ ZRdA ZSeTdBk r eR dS dS )CzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding
untokenize TokenInfoc * e Zd Zd Zed ZdS )r
c l d| j t | j fz }d| | z S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r tok_name_replace)selfannotated_types 5/opt/alt/python-internal/lib64/python3.11/tokenize.py__repr__zTokenInfo.__repr__/ s8 "di$)1D%EEJ
>
223 4 c p | j t k r | j t v rt | j S | j S N)r OPstringr r s r
exact_typezTokenInfo.exact_type4 s/ 9??t{.? ? ?$T[1 19 r N)__name__
__module__ __qualname__r propertyr r r r
r
. s> 4 4 4
X r r
ztype string start end linec 8 dd | z dz S )N(|))joinchoicess r groupr* ; s C#((7"3"3 3c 99r c t | dz S )Nr r* r( s r anyr- < s %/C//r c t | dz S )N?r, r( s r mayber0 = s E7Oc 11r z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c g d} dh}| D ]\}t j | D ]E}t j d |D D ]*}| d | +F]|S )N)brufbrfr c : g | ]}|| fS r" )upper).0cs r
z(_all_string_prefixes.._ s$ )D)D)DQ1aggii.)D)D)Dr )
_itertools permutationsproductaddr' )_valid_string_prefixesresultprefixtr4 s r _all_string_prefixesrF T s > = =TF( ' '(00 ' 'A ')D)D!)D)D)DE
'
'
2771::&&&&
' '
Mr c @ t j | t j S r )recompileUNICODE)exprs r _compilerL c s
:dBJ ' ''r z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c e Zd ZdS )
TokenErrorNr r r r" r r rR rR r rR c e Zd ZdS )StopTokenizingNrS r" r r rV rV rT r rV c & e Zd Zd Zd Zd Zd ZdS ) Untokenizerc > g | _ d| _ d| _ d | _ d S )N r )tokensprev_rowprev_colencodingr s r __init__zUntokenizer.__init__ s"
r c f |\ }}|| j k s|| j k r:|| j k r/t d ||| j | j || j z
}|r$| j d|z d| _ || j z
}|r | j d|z d S d S )Nz+start ({},{}) precedes previous end ({},{})\
r )r\ r]
ValueErrorformatr[ append)r startrowcol
row_offset
col_offsets r add_whitespacezUntokenizer.add_whitespace s S # "6"63 ;N;N J $fS#t}dm L LN N
N4=(
K v
2 3 3 3 DM4=(
1 K sZ / 0 0 0 0 0 1 1r c N t | }g }d}|D ]u}t | dk r| || nH|\ }}}} }
|t k r|| _ J|t
k r n |t k r| | y|t k r$| | \ | _
| _ |t t fv rd}nU|rS|rQ|d } |d t | k r.| j | t | | _ d}| | | j | | \ | _
| _ |t t fv r| xj
dz
c_
d| _ wd | j S )NF TrZ r r8 )iterlencompatENCODINGr^ ENDMARKERINDENTre DEDENTpopr\ r] NEWLINENLr[ rk r' ) r iterableitindents startlinerE tok_typetokenrf endlineindents r r zUntokenizer.untokenize s
(^^ " "A1vv{{ Ar"""01 -HeUC8## %
9$$6!!u%%%V##
/2,
t}gr]**
"w
" 8s6{{**K&&v...$'KKDM ! & & & K u % % %+. (DM4=GR=((
"
!
wwt{###r c g }| j j }|d t t fv }d}t j |g| D ]}|d d \ }} |t k r| | _ "|t t fv r| dz
} |t k r
|rd| z } d}nd}|t k r| | o|t k r|
|t t fv rd}n|r|r ||d d} || d S )Nr Frm rb Trn )r[ re rw rx r> chainrr r^ NAMENUMBERSTRINGrt ru rv )
r r~ ry r{ toks_appendr|
prevstringtoktoknumtokvals
r rq zUntokenizer.compat sD k( !H"
-
#UGX66 C !WNFF!! &
$ ''#
* 6\F !
"
v&&&6!!
GR=((
"w
" GBK((( ! K 9 r N)r r r r_ rk r rq r" r r rX rX sQ 1 1 1#$ #$ #$J" " " " " r rX c t }| | }|j | |j }|S r )rX r r^ encode)ry utouts r r r sC &
B
--
!
!C { jj%% Jr c | d d dd }|dk s| d rdS |dv s| d rdS | S ) N _-utf-8zutf-8-)zlatin-1
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r )lowerreplace
startswith)orig_encencs r _get_normal_namer sw 2
3B3-
'
'S
1
1C
g~~ 11~w
666
~~ABB 7| Or c j j n# t $ r d Y nw xY wdd }d} fd}fd} | }| t rd|dd }d}|s|g fS || }|r||gfS t
| s||gfS | }|s||gfS || }|r|||gfS |||gfS )NFr c < S # t $ r Y dS w xY wNr )
StopIterationreadlines r read_or_stopz%detect_encoding..read_or_stopC s6 8:: 33 s
c ( | d }n7# t $ r* d}d | }t | w xY wt | }|sd S t
| d } t | }n;# t $ r. d|z }nd | }t | w xY wr4|dk r)d}nd }t | |d z
}|S )
Nr z'invalid or missing encoding declarationz {} for {!r}rZ zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrd SyntaxError cookie_rematchr r* r LookupError)r line_stringmsgr r^ codec bom_foundfilenames r find_cookiez$detect_encoding..find_cookieI sX # ++g..KK! # # #;C##**399 c"" " # ,, 4#EKKNN33 #8$$EE # # # *X55<>%
\^^F {6""H )% (( UFO ##s ""c t | d } t |j \ }}| d t ||d }d|_ |S # | xY w)Nrbr T)line_bufferingr3 )
_builtin_openr r seekr modeclose)r bufferr^ linestexts r r r sx 8T
*
*F)&/::% A VXdCCC
s AA A1c t | \ }}t j d }t j |t | d | }t |j | S r )r r> repeatr ro _tokenize__next__)r r^ consumedemptyrl_gens r r r s[ $ )22Hh c " "E
hXs(;(;U
C
CF V_h / //r c
#
K dx}x}}d}d\ }}d }dg} |"|dk rd}t t |ddd V d}
d} | }
| } n# t $ r d} Y nw xY w|| | } |d
z
}dt | }
} |r| st d | | | }|rJ| d x} }t t || d | z |||f|| z V d\ }}d }n'|rN| d d d
k r@| dd dk r2t t || z ||t | f| V d}d }|| z }|| z } |dk r|s| snd}| |
k rO| | dk r|d
z
}n2| | dk r|t z d
z t z }n| | dk rd}nn | d
z
} | |
k O| |
k rnR| | dv r| | dk r]| | d d }t t ||| f|| t | z f| V | t | z
} t t | | d || f|t | f| V 4|| d k r;| | t t | d | |df|| f| V || d k rL|| vrt! dd|| | f | d d } t t" d|| f|| f| V || d k Ln| st d|df d}| |
k rt% t& | | }|r| d
\ }}||f||f|} }}||k r[| || | | }}||v s|dk r(|dk r"|dk r t t* |||| V nL|dv r>|dk r t t |||| V n&t t, |||| V n
|dk r t t |||| V n|t. v rt% t0 | }| | | }|r=| d } | || }t t |||| f| V np||f}| |d }| }ne|t2 v s"|d d t2 v s|d d t2 v r|d d k rz||f}t% t0 | p?t0 |d
p t0 |d }| |d d
}}| }nt t |||| V n| rt t8 |||| V nc|d k rd
}nZ|d v r|d
z
}n |d!v r|d
z }t t: |||| V n,t t | | || f|| d
z f| V | d
z
} | |
k |
rr|
d dvrh|
d sAt t, d|d
z
t |
f|d
z
t |
d
z fd V | d
d D ] }t t" d|df|dfd V !t t@ d|df|dfd V d S )"Nr
0123456789)r8 r r r )r r r8 r TrZ zEOF in multi-line stringra z\
rb z#
#z
rn z3unindent does not match any outer indentation levelz
zEOF in multi-line statement.z...rm r
\z([{z)]})!r
rr r r rp rR r r r
ERRORTOKENtabsizerstripCOMMENTrx re rt IndentationErrorru rL PseudoTokenspanr rw
triple_quotedendpats
single_quotedget isidentifierr r stripr rs )r r^ lnumparenlev continuednumcharscontstrneedcontcontliner{ last_liner posmaxstrstartendprogendmatchr column
comment_token pseudomatchrf sposeposr~ initialr s r r r s "##D#8iH GXHcG { " " H(FFB?????I D_
I8::DD DDD ;;x((D c$iiS ?
G !;XFFF }}T**H
$LLOO+c $tt*(< $ho? ? ? ? ?$)!
d233i6 1 1d233i86K6K
GdN#dCII%6B B B B B !D.#d?
]]9] F))9##aKFF#Y$&& $go 17 :FF#Y$&& FFq )) czzCyG##9##$(J$5$5f$=$=M#G] #;sS 5G5G/G(H$P P P P P3}---C DJ #;s4yy(94A A A A A ##v&&& TcT
T1Ic{DQQQQQ72;&&((*M%tS$79 9 9 "#2#, T3K$dKKKKK 72;&&
K !>q JJJICii";//55dC@@K L
( - -a 0 0
s#'-$cCdC<< !%eCi $u+wx'' ^^ % #FE4tDDDDDD && !|| 'E4t D DDDDD 'dD I IIIII ^^#GUD$EEEEEEm++ &wu~ 6 6G &}}T3 7 7H &ll1oo $U3Y 'tdC[$ O OOOOO$(%="&uvv,#' ..BQBi=00BQBi=00 RyD(($(%= #+7;;w+?+? ,A+2;;uQx+@+@,A+2;;uQx+@+@#B #B -1L!#' 'tT4 H HHHHH ))++ A#D%tTBBBBBB __ !II %'' A
E)) A
#BtT4@@@@@@
DI #;s1u
t= = = = =q] Ciic_D eYr]&00 9J9J9U9UVY9Z9Z0dQhI%?$(CPYNN]^L^A_acddddd!""+ > >T1Iay"======
IrD!9tQi
<
<<<<<.perrort s4
!!!
r c |r|f|z | fz } d|z n!|r |d| n d| z t j d d S )Nz%s:%d:%d: error: %sz : error: z error: %srZ )r exit)r r locationargsr s r errorzmain..errorx s * ;)WJ6D F(4/ 0 0 0 0
* Fhhh8 9 9 9 9 F;( ) ) ) r zpython -m tokenize)progr r/ z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zz %d,%d-%d,%d:2015rZ r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr r r rf r printr r r r rR r OSErrorKeyboardInterrupt Exception)r r parserr r r5 r[ r~
token_type token_rangeerrr r r s @r mainr p sl OOO
$
$*>
$
?
?F
Zs - F H H H igl G I I I D = 9}H x..
4! hqz 2 233
4
4
4
4
4
4
4
4
4
4
4
4
4
4
4 !H sy 1488F E EE Jz
. " -
(EK%),CDK ;; 4 4 4elllD
E
E
E
E E E 5 5 5x{1Q3' f
chqk8dF^444444444 5 5 5x{ f
chqk8dF^444444444
c8
c
o %+,,,
ss - D< "B9- D< 9B==D< B=A9D< <
H42E>>
H4 *F;;
H4 G
H4' G88 H4 H4 H//H4c # K dd l }| | D ]%}|\ }}}}}}} t ||||f||f| V &d S )Nr )r
TokenizerIterr
)
source c_tokenizerinfor r lineno
end_linenocol_off end_col_offr s
r !_generate_tokens_from_c_tokenizerr# s~ a####))&11 W WDHAT6:w TcFG#4z;6OQUVVVVVVW Wr __main__)U
__author__ __credits__builtinsr r codecsr r collections functoolsior itertoolsr> rH r r~ r rI ASCIIr r __all__
namedtupler
r* r- r0
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberrF lru_cacherL StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStr PseudoExtrasr r _prefixsetr r rE r4 rA r r rR rV rX r r r r r r
r r# r r" r r rR s , *
+ * * * * * # # # # # # # #
# # # # # # BJ@"( K K 2:0"( ; ;
- 6 6 6 6 & &{4PQQ : 9 9 / / / 1 1 1
cc*z122 2UU7^^ C
& . E)Y 9 = = (
U?)+ +-2U8__=
( *eJ))
U+[7-B
C
C
z; 2 2
( ( (
u**,,-
$ #
2
2 |e#\E%9 : : | == ==
?
? %RY'8$ G G GHH
I
h
U65&$
/
/
% ==c:&&' ==c:&&' ( ( u]GV44 55 vugtLLL
##%% ' 'G #GGcM #GGcM &GGeO &GGeO
A#gq3w
! %iU
# ! q
! ! ! ! ! ! ! ! % % % % %Y % % %\ \ \ \ \ \ \ \ ~ 6 Z$ Z$ Z$z
0 0 02t= t= t=n% % %; ; ;zW W W zDFFFFF r