4@54
I d Z ddlZddlmZ ddlmZ ddlmZ ddlm Z ddlm
Z
dd lm Z dd
lm Z dd l
mZ dd lmZ ed
Z ej dej Z ej d Z ej dej Z ej d Z ej dej ej z Z eddd ddl m Z dZ n # e $ r ej d Z dZ Y nw xY w e
d Z! e
d Z" e
d Z# e
d Z$ e
d Z% e
d Z& e
d Z' e
d! Z( e
d" Z) e
d# Z* e
d$ Z+ e
d% Z, e
d& Z- e
d' Z. e
d( Z/ e
d) Z0 e
d* Z1 e
d+ Z2 e
d, Z3 e
d- Z4 e
d. Z5 e
d/ Z6 e
d0 Z7 e
d1 Z8 e
d2 Z9 e
d3 Z: e
d4 Z; e
d5 Z< e
d6 Z= e
d7 Z> e
d8 Z? e
d9 Z@ e
d: ZA e
d; ZB e
d< ZC e
d= ZD e
d> ZE e
d? ZF e
d@ ZG e
dA ZH e
dB ZI e
dC ZJ e
dD ZK e
dE ZL e
dF ZM e
dG ZN e
dH ZO e
dI ZP e
dJ ZQi dKe!dLe9dMe%dNe(dOe1dPe0dQe4dRe:dSe,dTe6dUe-dVe7dWe+dXe5dYe'dZe2d[e)e*e.e/e"e&e#e3e$e8d\ ZR eSd] e eR D ZT eUeR eUeT k s
J d^ ej d_d`V da eWeRdb c D z ZX eYeGeIeHe;eLeMeNg ZZ eYe;eOeIeNg Z[dd Z\de Z]df Z^dg Z_dh Z` G di djea Zb G dk dlec Zde G dm dnea Zee G do dpea Zfdq Zg G dr dsec Zh G dt duea ZidS )vzImplements a Jinja / Python combination lexer. The ``Lexer`` class
is used to do some preprocessing. It filters out invalid operators like
the bitshift operators we don't allow in templates. It separates
template code and python code in expressions.
N) literal_eval)deque)
itemgetter )implements_iterator)intern) iteritems) text_type)TemplateSyntaxError)LRUCache2 z\s+z (\r\n|\r|\n)z7('([^'\\]*(?:\\.[^'\\]*)*)'|"([^"\\]*(?:\\.[^"\\]*)*)")z
(\d+_)*\d+z
(?eval)patternTz[a-zA-Z_][a-zA-Z0-9_]*Faddassigncoloncommadivdoteqfloordivgtgteqlbracelbracketlparenltlteqmodmulnepipepowrbracerbracketrparen semicolonsubtilde
whitespacefloatintegernamestringoperator block_begin block_endvariable_begin variable_end raw_beginraw_end
comment_begin comment_endcommentlinestatement_beginlinestatement_endlinecomment_beginlinecomment_end linecommentdatainitialeof+-/z//*%z**~[](){}z==z!=>) z>=r[ s BBBTQ1a&BBB zoperators droppedz(%s)rR c # > K | ]}t j | V d S N)reescaperW xs rZ rc s* UUqbillUUUUUUr\ c " t | S r^ )lenrb s rZ rg s SQRVVG r\ )keyc | t v r
t | S t dt dt dt dt
dt dt dt dt dt d t d
t d i
| | S ) Nzbegin of commentzend of commentr8 zbegin of statement blockzend of statement blockzbegin of print statementzend of print statementzbegin of line statementzend of line statementztemplate data / textzend of template)reverse_operatorsTOKEN_COMMENT_BEGINTOKEN_COMMENT_END
TOKEN_COMMENTTOKEN_LINECOMMENTTOKEN_BLOCK_BEGINTOKEN_BLOCK_ENDTOKEN_VARIABLE_BEGINTOKEN_VARIABLE_ENDTOKEN_LINESTATEMENT_BEGINTOKEN_LINESTATEMENT_END
TOKEN_DATA TOKEN_EOFget)
token_types rZ _describe_token_typery sz &&& ,, /+y951 8 4!#< !8*$
c*j!!
"r\ c X | j t k r| j S t | j S )z#Returns a description of the token.)type
TOKEN_NAMEvaluery )tokens rZ describe_tokenr s' zZ {
+ ++r\ c z d| v r'| dd \ }}|t k r|S n| }t | S )z0Like `describe_token` but for token expressions.rQ r )splitr| ry )exprr{ r} s rZ describe_token_exprr sL
d{{jja(( e : L % %%r\ c P t t | S )zsCount the number of newline characters in the string. This is
useful for extensions that filter a stream.
)re
newline_refindall)r} s rZ count_newlinesr s z!!%(( ) ))r\ c f t j }t | j t || j ft | j t || j ft | j t || j fg}| j @|
t | j t d || j z f | j @|
t | j t d || j z f d t |d D S )zACompiles all the rules from the environment into a list of rules.Nz ^[ \t\v]*z(?:^|(?<=\S))[^\S\r\n]*c " g | ] }|d d
S )r NrV ra s rZ r[ z!compile_rules.. s 7 7 7aAabbE 7 7 7r\ T)reverse)r_ r` re comment_start_stringrk block_start_stringro variable_start_stringrq line_statement_prefixappendrs line_comment_prefixTOKEN_LINECOMMENT_BEGINsorted) environmenteruless rZ
compile_rulesr sC
A
0 1 1
Ak. / /
. / /
Ak, - -
1 2 2
Ak/ 0 0
E$ (4
K566) qq!B C CC
&2
K344'*QQ{/N-O-OO
8 76%666 7 7 77r\ c " e Zd ZdZefdZd ZdS )FailurezjClass that raises a `TemplateSyntaxError` if called.
Used by the `Lexer` to specify known errors.
c " || _ || _ d S r^ )message error_class)selfr clss rZ __init__zFailure.__init__ s r\ c : | | j || r^ )r r )r linenofilenames rZ __call__zFailure.__call__ s t|VX>>>r\ N)__name__
__module__ __qualname____doc__r r r rV r\ rZ r r sF %8 ? ? ? ? ?r\ r c f e Zd ZdZdZd ed D \ ZZZd Z d Z
d Z d Z d Z
d
S ) Tokenz Token class.rV c # N K | ] }t t | V !d S r^ )propertyr ra s rZ rc zToken. s0 EEq8JqMM22EEEEEEr\ c r t | |t t | |f S r^ )tuple__new__r str)r r r{ r} s rZ r z
Token.__new__ s+ }}S66#d))+<+EL'IM &%/3tt^DL5Q5Q5QR #
< JJJJDJJJJs %B; ;C N)r )r r r r r r r __nonzero__r r r r r r r r r r rV r\ rZ r r @ s
) ) )H H H K
X# # #
. . .
r\ r c | j | j | j | j | j | j | j | j | j | j | j
| j f }t
| }|t | }|t |< |S )z(Return a lexer which is probably cached.)r block_end_stringr variable_end_stringr comment_end_stringr r trim_blocks
lstrip_blocksnewline_sequencekeep_trailing_newline _lexer_cacherw Lexer)r rh lexers rZ get_lexerr s &$)'(&)' !$)
C
S ! !E }k"" ! S Lr\ c & e Zd ZdZdZ fdZ xZS )OptionalLStripzWA special tuple for marking a point in the state that can have
lstrip applied.
rV c V t t | | | S r^ )superr r )r memberskwargs __class__s rZ r zOptionalLStrip.__new__ s# ^S))11#w???r\ )r r r r r r
__classcell__)r s @rZ r r sR I@ @ @ @ @ @ @ @ @r\ r c 6 e Zd ZdZd Zd ZddZd dZd dZdS )
r a
Class that implements a lexer for a given environment. Automatically
created by the environment class, usually you don't have to do that.
Note that the lexer is not automatically bound to an environment.
Multiple environments can share the same lexer.
c t j }d }t t d ft t
d ft t d ft t d ft t d ft t d fg}t | }|j rdpd}|j r |d nd | _ |j | _ |j | _ d |dd d ||j d ||j d
||j d gd |D z z t/ t0 d
d
f |d t0 d fgt2 |d ||j d
||j d|d t6 t8 fdf |d t; d fd fgt< |d ||j d
||j d| t> dfg|z t@ |d ||j! d
||j! tD dfg|z tF |d ||j d ||j d
||j |d t/ t0 tH df |d t; d fd fgtJ |d tL dfg|z tN |d tP tR fdfgi| _* d S )Nc Z t j | t j t j z S r^ )r_ compileMSrf s rZ czLexer.__init__..c s :a-- -r\ z\n?r z[^ \t]rootz (.*?)(?:%s)rR z(?Pz(\-|\+|)\s*raw\s*(?:\-z\s*|z))c & g | ]\ }}d |d|dS )z(?P. s; $(Aqq :;AAA > r\ #bygroupz.+z (.*?)((?:\-rJ #popz(.)zMissing end of comment tagz(?:\-z\-z (.*?)((?:z(\-|\+|))\s*endraw\s*(?:\-z Missing end of raw directivez \s*(\n|$)z(.*?)()(?=\n|$))+r_ r`
whitespace_reTOKEN_WHITESPACEfloat_re TOKEN_FLOAT
integer_re
TOKEN_INTEGERname_rer| string_re TOKEN_STRING operator_reTOKEN_OPERATORr r r lstrip_unless_rer r joinr r r ru rk r rm rl r ro rp rq r rr TOKEN_RAW_BEGIN
TOKEN_RAW_ENDrs rt r rn TOKEN_LINECOMMENT_ENDr )r r r r tag_rulesroot_tag_rulesblock_suffix_res rZ r zLexer.__init__ s I . . .
,d 3
{D )
-
j$ '
d +
.$ /
'{33 &1 > > > Ak = > > > '
#
#
AA Ak < = = = = Ak : ; ; ; ; Ak : ; ; ; +OO #:}== 5G$B C CEtL "
&< "96B( (
$A())&(=>&Ch
r\ c B t | j | S )z@Called for strings and template data to normalize it to unicode.)r r( r )r r} s rZ _normalize_newlineszLexer._normalize_newlinesa s ~~d 3U;;;r\ Nc ~ | |||| }t | ||| || S )z:Calls tokeniter + tokenize and wraps it in a token stream.) tokeniterr wrap)r sourcer- r stater s rZ tokenizezLexer.tokenizee s; h>>499VT8<t | }t r&| st d||| n%|t k r | |dd dd d }n# t" $ rQ}t | d d }t |||| d }~ww xY w|t( k r$t+ | d
d }nG|t. k r$t1 | d
d }n|t2 k r
t4 | }t7 ||| V d S ) zThis is called with the stream as returned by `tokenize` and wraps
every token in a :class:`Token` and converts the value.
keywordz Invalid character in identifierr asciibackslashreplacezunicode-escaperQ Nr r ) ignored_tokensrs ro rt rp r r ru r# r| r check_ident isidentifierr r encodedecode Exceptionr stripr intreplacer r r operatorsr ) r r r- r r r~ r} r msgs rZ r& z
Lexer.wrapj s# %+ & . & . FE5&&333)111'?M:::*$$ 0077)## *$$E
u'9'9';'; -964 ,&&K 00qt== );<< 011 E
! K K K a&&,,s++B/5577C-c64JJJK -''EMM#r 2 233+%%$U]]3%;%;<<.((!%(u-- - - - -M& . & .s AD
E0 A E++E0c # 8
K t | }| }| j r3|r1dD ].}| | r| d n/d | }d}d}dg} |,|dk r&|dv s
J d | |d
z | j | d }
t | } g } | j }
d}d } |
D ]\ }}}| || }| | r|t t t fv r9t |t r| }t |t r|d }t# d
|ddd D }|dk rM| }|t | d d }|f|dd z }n|dk rz|
x| t, sL| d dz }|dk s|r,|
|| s|d| f|dd z }t3 | D ]\ }}|j t6 u r ||| |dk r]t9 | D ](\ }} | !||| fV || d z
} n)t; d|z || } | s |t< vr||| fV || d |z z
}d}n| } |t@ k r| dk r| d n| dk r| d nj| dk r| d nN| dv rJ| stC d| z ||| | " } | | k rtC d| d | d ||| | s |t< vr||| fV || d z
}| d d dk }|# } ||d k r| " nn|dk rSt9 | D ] \ }} | | | n t; d |z n| | | j | d }
n| |k rt; d |z | } n%|| k rdS tC d!|| |fz ||| B)"zThis method tokenizes the text and returns the tokens in a
generator. Use this method if you just want to tokenize a template.
)z
r r< r r r
N)variableblockz
invalid state_beginr, Tc 3 K | ]}||V d S r^ rV )rW gs rZ rc z"Lexer.tokeniter.. s" )S)SQ]!]]]])S)Sr\ rB rA r
z?%r wanted to resolve the token dynamically but no group matchedrK rL rI rJ rG rH )rL rJ rH zunexpected '%s'z unexpected 'z
', expected ''r zC%r wanted to resolve the new state dynamically but no group matchedz,%r yielded empty string without stack changezunexpected char %r at %d)$r
splitlinesr endswithr r r re r matchrr rp rt
isinstancer groupsr r rstripcount groupdictrw rq rfindsearch enumerater r r RuntimeErrorignore_if_emptygroupr r popend) r r' r- r r( linesnewlineposr stack statetokens
source_lengthbalancing_stackr newlines_stripped
line_startingregextokens new_statemrH text
strip_signstrippedl_posidxr~ rh r} r> expected_oppos2s rZ r% zLexer.tokeniter sq 6""!!## % & /
??7++ LL$$$E 5!! & 1111?111 LL) * * *jr+ F
0
a ,7 _
_
(vyKK,,9 # v&#+2 ( (
fe,, Y/ XXZZF!&.99 J &ay
&*)S)SVADqD\)S)S)S%S%S
%,,'+{{}}H04S]]__0E0K0KD0Q0Q -&.[6!"":%=FF '# - - 0 <$%KKMM$5$56J$K$K !=
%)JJt$4$4q$8E $qyyMy (8'>'>tU'K'K !J.26E6l_vabbz-IF&/&7&7 2 2
U ?g55"'%"9"9 9 #j 0 0.7
.F.F
"
"
U#(#4*0#u*<$<$<$<$*ekk$.?.?$?F$)E $5
'3%<>C%D'" '" !" %* $*#;D # :uO'C'C&,eT&9 9 9 9 "djj&6&69J&J JF01 - -32: 7799D // 3;; + 2 23 7 7 7 7 !S[[ + 2 23 7 7 7 7 !S[[ + 2 23 7 7 7 7 !_ 4 4#2 "&9$5$r s
( ( ( ( ( ( + + + + + + x||
624((
RZ
(
(
BJBBD
RZ
&
&
2: MBJ GG[&))) 0/////KK bj233GKKK
F5MM vh fWoo fWoo F5MM F5MM 6$<<
## 6$<<
VF^^
vh
##vh 6$<<
VF^^
F5MM F5MM 6$<<
VF^^
F5MM vh
##vh &%% F5MM fWoo 6,'' fWoo y!!
VF^^
vh
##F=)) &%% v .// VN++ &%%y!!
f_-- F=)) y!!
"F#8 9 9 &!455 &!455 011 F=))
VF^^
y!!
F5MM
.
)
(
(!
" #
$
5
: DBBYYy-A-ABBBCC
s9~~ ./////1D///bj
SXXUUFF9BSBS,T,T,TUUU
U
UU
)z=2CD
" " "&, , ,& & &* * *&8 &8 &8R
?
?
?
?
?f
?
?
?%J %J %J %J %JE %J %J %JP & ( c c c c c& c c cL .
@
@
@
@
@U
@
@
@E E E E EF E E E E Es 3
C C#"C#