B
sa; / @ s d Z dZddlmZ ddlmZmZ ddlZddl m
Z
ddl m Z ddl Z
ddlZddlZddlT ed ejZed
ejZddlZejd d d
dg Z[eeeeeee e e e e e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;ee?e@eAeBeCd.ZDG dd deEddZFdd ZGdd ZHdd ZIdZJdZKeJeHdeJ eIeK ZLdZMd ZNd ZOd ZPd ZQeGeNeOePeQZRd ZSeGd!d"eIeS ZTd#eS ZUeGeTeUZVeGd$eVd% ZWeGeWeVeRZXd&d' ZYd(d) ZZeGeY Z[d*Z\d+Z]d,Z^d-Z_eGe[d. e[d/ Z`eGe[d0 e[d1 ZaeGd2d3d4d5d6d7d8d9Zbd:ZceGd;d eGd?d e[d@ eGdAd ZheGdBeKe`ZieJeGeieXeeeheM Zji Zkx@eY D ]6Zle\ekeld? < e]ekeldA < e^ekeld. < e_ekeld/ < qW em Znem Zox\eY D ]RZpx$epdA epd? fD ]Zqenreq qW x$epd/ epd. fD ]Zqeoreq q$W qW dCZsG dDdE dEetZuG dFdG dGetZvG dHdI dIZwdJd
ZxdKdL ZydMd ZzdNdO ZdPd Z{dQdR Z|dSdT Z}dUdV Z~edWkre~ dS )XzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)tokenizedetect_encoding
untokenize TokenInfo).()[]:,;+-r /|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z...z->@z@=c @ s e Zd Zdd Zedd ZdS )r c C s$ d| j t| j f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r" tok_name_replace)selfannotated_type r' -/opt/alt/python37/lib64/python3.7/tokenize.py__repr___ s zTokenInfo.__repr__c C s( | j tkr | jtkr t| j S | j S d S )N)r" OPstringEXACT_TOKEN_TYPES)r% r' r' r(
exact_typed s
zTokenInfo.exact_typeN)__name__
__module__ __qualname__r) propertyr- r' r' r' r( r ^ s ztype string start end linec G s dd | d S )Nr r r
)join)choicesr' r' r( groupk r4 c G s t | d S )Nr )r4 )r3 r' r' r( anyl r5 r6 c G s t | d S )N?)r4 )r3 r' r' r( maybem r5 r8 z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sj ddddddg} dh}xN| D ]F}x@t |D ]2}x,t jdd |D D ]}|d| qFW q,W q W |S )
Nbrufbrfr c S s g | ]}|| fqS r' )upper).0cr' r' r(
s z(_all_string_prefixes..)
_itertools permutationsproductaddr2 )_valid_string_prefixesresultprefixtr; r' r' r( _all_string_prefixes s
rL c C s t | t jS )N)recompileUNICODE)exprr' r' r( _compile s rQ z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c @ s e Zd ZdS )
TokenErrorN)r. r/ r0 r' r' r' r( rU s rU c @ s e Zd ZdS )StopTokenizingN)r. r/ r0 r' r' r' r( rV s rV c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d | _d S )N r )tokensprev_rowprev_colencoding)r% r' r' r( __init__ s zUntokenizer.__init__c C s |\}}|| j k s&|| j kr>|| jk r>td||| j | j|| j }|rb| jd| d| _|| j }|r| jd| d S )Nz+start ({},{}) precedes previous end ({},{})z\
r )rZ r[
ValueErrorformatrY append)r% startrowcol
row_offset
col_offsetr' r' r( add_whitespace s
zUntokenizer.add_whitespacec C s4 t |}g }d}x|D ]
}t|dkr8| || P |\}}}} }
|tkrV|| _q|tkr`P |tkrv|| qnl|tkr| | \| _
| _ qnL|t t
fkrd}n:|r|r|d } |d t| kr| j| t| | _ d}| | | j| | \| _
| _ |t t
fkr| j
d7 _
d| _ qW d| jS )NF TrX r r? )iterlencompatENCODINGr\ ENDMARKERINDENTra DEDENTpoprZ r[ NEWLINENLrY rg r2 ) r% iterableitindents startlinerK tok_typetokenrb endlineindentr' r' r( r
sF
zUntokenizer.untokenizec
C s g }| j j}|d ttfk}d}xt|g|D ]}|d d \}} |tkrR| | _q.|ttfkrf| d7 } |t kr|rzd| } d}nd}|t
kr|| q.n>|t kr| q.n*|ttfkrd}n|r|r||d d}|| q.W d S )Nr Frh r^ Tri )
rY ra rr rs r rm r\ NAMENUMBERSTRINGro rp rq )
r% ry rt rv toks_appendrw
prevstringtoktoknumtokvalr' r' r( rl s8
zUntokenizer.compatN)r. r/ r0 r] rg r
rl r' r' r' r( rW s
%rW c C s* t }|| }|jd k r&||j}|S )N)rW r
r\ encode)rt utoutr' r' r( r
8 s
c C sH | d d dd}|dks*|dr.dS |dks@|drDdS | S ) N _r zutf-8zutf-8-)zlatin-1z
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-z
iso-8859-1)lowerreplace
startswith)orig_encencr' r' r( _get_normal_nameS s
r c s y j jW n tk
r$ d Y nX d d }d}fdd} fdd}| }|trpd |dd }d }|s||g fS ||}|r||gfS t|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c s y S t k
r dS X d S )Nr5 )
StopIterationr' )readliner' r( read_or_stopv s z%detect_encoding..read_or_stopc s y| d}W n4 tk
rB d}d k r6d|}t|Y nX t|}|sVd S t|d}y t|}W n: t k
r d krd| }n d|}t|Y nX r|dkr؈d krd}n
d}t||d 7 }|S )
Nzutf-8z'invalid or missing encoding declarationz {} for {!r}rX zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorr` SyntaxError cookie_rematchr r4 r LookupError)r{ line_stringmsgr r\ codec) bom_foundfilenamer' r( find_cookie| s6
z$detect_encoding..find_cookieT z utf-8-sig)__self__nameAttributeErrorr r blank_rer )r r\ defaultr r firstsecondr' )r r r r( r ^ s8
&
c C sV t | d}y0t|j\}}|d t||dd}d|_|S | Y nX d S )Nrbr T)line_bufferingr: )
_builtin_openr r seekr modeclose)r bufferr\ linestextr' r' r( r s
r c C sB ddl m}m} t| \}}t| d}|d}t||||j|S )Nr )r repeatr5 ) itertoolsr r r rj _tokenize__next__)r r r r\ consumedrl_genemptyr' r' r( r s
c c s d } }}d}d\}}d }dg} |d k rH|dkr6d}t t|dddV d}
d} xy| }
| } W n tk
rz d} Y nX |d k r| |} |d 7 }dt| } }
|rt| std
||| }|r
|d } }t t|| d | |||f|| V d\}}d }nf|r^| d d d kr^| d
d dkr^t t || ||t| f|V d}d }qTn|| }|| }qTn|dkr |s | sP d}xf| |
k r| | dkr|d 7 }n6| | dkr|t
d t
}n| | dkrd}nP | d 7 } qW | |
krP | | dkr| | dkr`| | d d}t t ||| f|| t| f| V | t|7 } t t
| | d || f|t| f| V qT|| d kr| | t t| d | |df|| f| V xp|| d k r|| krtdd|| | f| d d } t td|| f|| f| V qW n| s0td|dfd}x| |
k rtt| | }|r|d \}}||f||f| }}} ||krq8| || | | }}||ks|dkr|dkr|dkrt t|||| V q|dkr |dkrt t
|||| V nt t|||| V q|dkr>t t |||| V q|tkrtt| }|| | }|r|d} | || }t t|||| f| V n||f}| |d }| }P q|tks|d d tks|d d tkrR|d d kr>||f}tt|p t|d p t|d }| |d d }}| }P nt t|||| V nf| rpt t |||| V nH|d krd }n8|d kr|d 7 }n|d kr|d 8 }t t |||| V n*t t | | || f|| d f| V | d 7 } q8W qTW |
r0|
d dkr0t td|d t|
f|d t|
d fdV x0| d d D ] }t td|df|dfdV q>W t t d|df|dfdV d S )!Nr
0123456789)r? r z utf-8-sigzutf-8)r r r? r5 rX zEOF in multi-line stringz\
z\
r^ z#
#z
ri z3unindent does not match any outer indentation levelz
zEOF in multi-line statementr z...rh r
\z([{z)]}) r rm r r rk rU r rz r
ERRORTOKENtabsizerstripCOMMENTrs ra ro IndentationErrorrp rQ PseudoTokenspanr~ rr
triple_quotedendpats
single_quotedget isidentifierr} r* rn )r r\ lnumparenlev continuednumcharscontstrneedcontcontlinerv last_liner{ posmaxstrstartendprogendmatchrz column
comment_token pseudomatchrb sposeposry initialr| r' r' r( r s
*
"
"
. r c C s
t | d S )N)r )r r' r' r( generate_tokens s r c
s( dd l } dd d fdd }| jdd}|jdd d
d d |jd
ddddd | }y|jr|j}t|d}tt|j}W d Q R X nd}t t
j jd }xF|D ]>}|j }|j
r|j}d|j|j } td| t| |jf qW W n8 tk
r: }
z0|
jd dd \} } ||
jd || | f W d d }
~
X Y n tk
r }
z(|
jd \} } ||
jd || | f W d d }
~
X Y n tk
r }
z||
| W d d }
~
X Y nx tk
r }
z ||
W d d }
~
X Y nN tk
r td Y n2 tk
r" }
z d|
W d d }
~
X Y nX d S ) Nr c S s t | tjd d S )N)file)printsysstderr)messager' r' r( perror s zmain..perrorc sR |r"|f| | f } d| n"|r8 d|| f n d| t d d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %srX )r exit)r r locationargs)r r' r( error s zmain..errorzpython -m tokenize)progr r7 z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zz %d,%d-%d,%d:z%-20s%-15s%-15rrX r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr" r r- rb rz r r# r+ r r rU r OSErrorKeyboardInterrupt Exception)
r r parserr r r< rY ry
token_type token_rangeerrr{ r r' )r r( main sN
&& r __main__)
__author__ __credits__builtinsr r codecsr r collectionsior r r rD rM r ry rN ASCIIr r __all__LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALELLIPSISRARROWATATEQUALr,
namedtupler r4 r6 r8
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberrL rQ StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtrasr r _prefixsetr r rK r; rG r r rU rV rW r
r r r r r r r. r' r' r' r( s
_ ] :<