a
lja~ @ s d dl mZmZmZ d dlmZ d dlmZmZ d dl Z d dl
Z
d dl m Z m
Z
d dlmZ ddlmZmZmZmZ dd lmZ dd
lmZ ed d eD Zed
d eD Zedd eD ZeeddgB Z dZ ej r e d dkre ddksJ e
e dd e!d d Z"n
e
e Z"h dZ#e
dZ$i Z%G dd de&Z'dd Z(G dd d e&Z)G d d d e)Z*G d d d e+Z,G d!d" d"e&Z-G d#d$ d$e&Z.d%d& Z/dS )' )absolute_importdivisionunicode_literals) text_type) http_clienturllibN)BytesIOStringIO) webencodings )EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utilsc C s g | ]}| d qS asciiencode.0item r E/usr/lib/python3.9/site-packages/pip/_vendor/html5lib/_inputstream.py
r c C s g | ]}| d qS r r r r r r r r c C s g | ]}| d qS r r r r r r r r > i
i i i i i i i i i i i i i i i i i i i
i i i i i i i
i
i i i i z[ -
-/:-@\[-`{-~]c @ sH e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dS )BufferedStreamzBuffering for streams that do not have buffering of their own
The buffer is implemented as a list of chunks on the assumption that
joining many strings will be slow since it is O(n**2)
c C s || _ g | _ddg| _d S )Nr r )streambufferposition)selfr" r r r __init__: s zBufferedStream.__init__c C s< d}| j d | jd D ]}|t|7 }q|| jd 7 }|S Nr r )r# r$ len)r% poschunkr r r tell? s
zBufferedStream.tellc C sT || ksJ |}d}t| j| |k rF|t| j| 8 }|d7 }q||g| _d S r' )_bufferedBytesr( r# r$ )r% r) offsetir r r seekF s
zBufferedStream.seekc C sT | j s| |S | jd t| j krF| jd t| j d krF| |S | |S d S )Nr r r )r# _readStreamr$ r( _readFromBufferr% bytesr r r readO s
zBufferedStream.readc C s t dd | jD S )Nc S s g | ] }t |qS r )r( r r r r r Y r z1BufferedStream._bufferedBytes..)sumr# r% r r r r, X s z BufferedStream._bufferedBytesc C s< | j |}| j| | jd d7 < t|| jd< |S r' )r" r4 r# appendr$ r( )r% r3 datar r r r0 [ s
zBufferedStream._readStreamc C s |}g }| j d }| j d }|t| jk r|dkr|dks>J | j| }|t|| krl|}||| g| _ n"t|| }|t|g| _ |d7 }||||| ||8 }d}q |r|| | d|S )Nr r r )r$ r( r# r7 r0 join)r% r3 ZremainingBytesrvZ bufferIndexZ bufferOffsetZ bufferedDataZ bytesToReadr r r r1 b s&
z BufferedStream._readFromBufferN) __name__
__module__ __qualname____doc__r& r+ r/ r4 r, r0 r1 r r r r r! 3 s r! c K s t | tjs(t | tjjr.t | jtjr.d}n&t| drJt | dt }n
t | t }|rdd |D }|rvt
d| t | fi |S t | fi |S d S )NFr4 r c S s g | ]}| d r|qS ) _encoding)endswith)r xr r r r r z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r)
isinstancer Z HTTPResponser ZresponseZaddbasefphasattrr4 r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargsZ isUnicode encodingsr r r HTMLInputStream} s
rK c @ sp e Zd ZdZdZdd Zdd Zdd Zd d
Zd d Z d
d Z
dddZ dd Z dd Z
d ddZdd ZdS ) rF Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
i ( c C sZ t jsd| _n tddkr$| j| _n| j| _dg| _tddf| _| || _
| dS ) Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
Nu r r utf-8certain) r supports_lone_surrogatesreportCharacterErrorsr( characterErrorsUCS4characterErrorsUCS2ZnewLineslookupEncoding charEncoding
openStream
dataStreamreset)r% rH r r r r& s
z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N r )r* chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacterr6 r r r rX s z HTMLUnicodeInputStream.resetc C s t |dr|}nt|}|S zvProduces a file object from source.
source can be either a file object, local filename or a string.
r4 )rD r r% rH r" r r r rV s
z!HTMLUnicodeInputStream.openStreamc C sT | j }|dd|}| j| }|dd|}|dkr@| j| }n ||d }||fS )N
r r r )r* countr] rfindr^ )r% r- r* ZnLinesZ positionLineZ lastLinePosZpositionColumnr r r _position s
z HTMLUnicodeInputStream._positionc C s | | j\}}|d |fS )z:Returns (line, col) of the current position in the stream.r )re r[ )r% linecolr r r r$ s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return
EOF when EOF is reached.
r )r[ rZ readChunkr r* )r% r[ charr r r ri s
zHTMLUnicodeInputStream.charNc C s |d u r| j }| | j\| _| _d| _d| _d| _| j|}| j rX| j | }d | _ n|s`dS t
|dkrt |d }|dksd| krdkrn n|d | _ |d d }| j r| | |
d d
}|
d d
}|| _t
|| _d S )
NrY r Fr r
i z
rb
T)_defaultChunkSizere rZ r] r^ r* r[ rW r4 r_ r( ordrQ replace)r% rZ r8 Zlastvr r r rh s0
z HTMLUnicodeInputStream.readChunkc C s( t tt|D ]}| jd qd S )Ninvalid-codepoint)ranger( invalid_unicode_refindallr\ r7 )r% r8 _r r r rR s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s d}t |D ]}|rqt| }| }t|||d rrt|||d }|tv rl| j
d d}q|dkr|dkr|t |d kr| j
d qd}| j
d qd S )NF rp Trk i r ) rr finditerrn groupstartr ZisSurrogatePairZsurrogatePairToCodepointnon_bmp_invalid_codepointsr\ r7 r( )r% r8 skipmatchZ codepointr) Zchar_valr r r rS # s" z*HTMLUnicodeInputStream.characterErrorsUCS2Fc
C s zt ||f }W nf tyv |D ]}t|dk s"J q"ddd |D }|sXd| }td| }t ||f< Y n0 g }|| j| j}|du r| j| j krqn0|
}|| j kr| | j| j| || _q| | j| jd | s|qq|d|} | S )z Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
rY c S s g | ]}d t | qS )z\x%02x)rn )r cr r r r H r z5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N)
charsUntilRegExKeyErrorrn r9 recompiler{ r* r[ rZ endr7 rh )
r% Z
charactersZoppositecharsr} Zregexr: mr rr r r
charsUntil: s0
z!HTMLUnicodeInputStream.charsUntilc C sT |t urP| jdkr.|| j | _| jd7 _n"| jd8 _| j| j |ksPJ d S r' )r r[ r* rZ )r% ri r r r ungeti s
z HTMLUnicodeInputStream.unget)N)F)r; r< r= r> rm r& rX rV re r$ ri rh rR rS r r r r r r rF s
&
/rF c @ sL e Zd ZdZdddZdd Zd d
Zdd d Zd
d Zdd Z dd Z
dS )rG rL N windows-1252Tc C sn | || _t| | j d| _d| _|| _|| _|| _|| _ || _
| || _ | j d dusbJ |
dS )rM i d r N)rV rawStreamrF r& numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrU rX )r% rH r r r r r Z
useChardetr r r r& s z HTMLBinaryInputStream.__init__c C s& | j d j| jd| _t| d S )Nr ro )rU Z
codec_info streamreaderr rW rF rX r6 r r r rX s zHTMLBinaryInputStream.resetc C sJ t |dr|}nt|}z|| W n tyD t|}Y n0 |S r` )rD r r/ r+ Exceptionr! ra r r r rV s
z HTMLBinaryInputStream.openStreamc C s | df}|d d ur |S t| jdf}|d d ur:|S t| jdf}|d d urX|S | df}|d d urt|S t| jdf}|d d ur|d jds|S t| jdf}|d d ur|S |rnzddl m
} W n t y Y n0 g }| }|j s:| j
| j}t|tsJ |s$q:|| || q| t|jd }| j
d |d urn|dfS t| jdf}|d d ur|S tddfS )NrO r Z tentativezutf-16)UniversalDetectorencodingr ) detectBOMrT r r detectEncodingMetar name
startswithr Z%pip._vendor.chardet.universaldetectorr ImportErrorZdoner r4 r rB r3 r7 Zfeedcloseresultr/ r )r% ZchardetrU r ZbuffersZdetectorr# r r r r r sR
z'HTMLBinaryInputStream.determineEncodingc C s | j d dksJ t|}|d u r&d S |jdv rFtd}|d usJ nT|| j d krf| j d df| _ n4| jd |df| _ | td| j d |f d S )Nr rO utf-16beutf-16lerN r z Encoding changed from %s to %s)rU rT r r r/ rX r )r% Z newEncodingr r r changeEncoding s
z$HTMLBinaryInputStream.changeEncodingc
C s t jdt jdt jdt jdt jdi}| jd}t|t s r& rX rV r r r r r r r r rG z s
*
>"rG c @ s e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z e e e
Z
dd Ze eZefddZdd Zdd Zdd Zd S )
EncodingByteszString-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raisedc C s t |tsJ t| | S N)rB r3 __new__lowerr% valuer r r r F s zEncodingBytes.__new__c C s
d| _ d S )Nr )re r r r r r& J s zEncodingBytes.__init__c C s | S r r r6 r r r __iter__N s zEncodingBytes.__iter__c C s> | j d }| _ |t| kr"tn |dk r.t| ||d S )Nr r re r(
StopIterationrE r% pr r r __next__Q s zEncodingBytes.__next__c C s | S r )r r6 r r r nextY s zEncodingBytes.nextc C sB | j }|t| krtn |dk r$t|d | _ }| ||d S r' r r r r r previous] s zEncodingBytes.previousc C s | j t| krt|| _ d S r re r( r )r% r$ r r r setPositionf s zEncodingBytes.setPositionc C s* | j t| krt| j dkr"| j S d S d S )Nr r r6 r r r getPositionk s
zEncodingBytes.getPositionc C s | | j | j d S Nr )r$ r6 r r r getCurrentByteu s z EncodingBytes.getCurrentBytec C sH | j }|t| k r>| ||d }||vr4|| _|S |d7 }q|| _dS )z Skip past a list of charactersr Nr$ r( re r% r r r} r r r rz z s
zEncodingBytes.skipc C sH | j }|t| k r>| ||d }||v r4|| _|S |d7 }q|| _d S r r r r r r skipUntil s
zEncodingBytes.skipUntilc C s( | || j}|r$| jt|7 _|S )zLook for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone)r r$ r( )r% r3 r: r r r
matchBytes s zEncodingBytes.matchBytesc C s< z | || jt| d | _W n ty6 tY n0 dS )zLook for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the matchr T)indexr$ r( re
ValueErrorr r2 r r r jumpTo s
zEncodingBytes.jumpToN)r; r< r= r> r r& r r r r r r propertyr$ r currentBytespaceCharactersBytesrz r r r r r r r r B s
r c @ sX e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z dd Z dS )r z?Mini parser for detecting character encoding from meta elementsc C s t || _d| _dS )z3string - the data to work on for encoding detectionN)r r8 r r% r8 r r r r& s
zEncodingParser.__init__c
C s d| j vrd S d| jfd| jfd| jfd| jfd| jfd| jff}| j D ]}d}z| j d W n tyx Y qY n0 |D ]B\}}| j |r~z| }W qW q~ ty d}Y qY q~0 q~|sH qqH| j S ) Ns r8 r r6 r r r r s z EncodingParser.handleCommentc C s | j jtvrdS d}d }| }|d u r,dS |d dkr\|d dk}|r|d ur|| _dS q|d dkr|d }t|}|d ur|| _dS q|d dkrtt|d }| }|d urt|}|d ur|r|| _dS |}qd S ) NTFr s
http-equivr s content-type charsets content) r8 r r getAttributer rT ContentAttrParserr parse)r% Z hasPragmaZpendingEncodingattrZtentativeEncodingcodecZ
contentParserr r r r s8 zEncodingParser.handleMetac C s
| dS )NF)handlePossibleTagr6 r r r r s z%EncodingParser.handlePossibleStartTagc C s t | j | dS )NT)r r8 r r6 r r r r s
z#EncodingParser.handlePossibleEndTagc C sb | j }|jtvr(|r$| | dS |t}|dkrD| n| }|d ur^| }qLdS )NTr )r8 r asciiLettersBytesr r r spacesAngleBracketsr )r% ZendTagr8 r} r r r r r s
z EncodingParser.handlePossibleTagc C s | j dS )Nr r r6 r r r r s zEncodingParser.handleOtherc C s | j }|ttdgB }|du s2t|dks2J |dv r>dS g }g }|dkrV|rVqnX|tv rj| }qnD|dv rd|dfS |tv r|| n|du rdS || t |}qF|dkr|
d|dfS t | | }|dv rJ|}t |}||kr"t | d|d|fS |tv r<|| q|| qnJ|d krbd|dfS |tv r||| n|du rdS || t |}|t v rd|d|fS |tv r|| n|du rdS || qdS )
z_Return a name,value pair for the next attribute in the stream,
if one is found, or None /Nr )r N =)r r r ) ' "r ) r8 rz r frozensetr( r9 asciiUppercaseBytesr7 r r r r )r% r8 r} ZattrNameZ attrValueZ quoteCharr r r r sb
zEncodingParser.getAttributeN)
r; r< r= r> r& r r r r r r r r r r r r r s $r c @ s e Zd Zdd Zdd ZdS )r c C s t |tsJ || _d S r )rB r3 r8 r r r r r& a s zContentAttrParser.__init__c C s z| j d | j jd7 _| j | j jdks sH
J g Ib ='