a
lja~ @ s d dl mZmZmZ d dlmZ d dlmZmZ d dl Z d dl
Z
d dl m Z m
Z
d dlmZ ddlmZmZmZmZ dd lmZ dd
lmZ ed d eD Zed
d eD Zedd eD ZeeddgB Z dZ ej re
e dd e d d Z!n
e
e Z!h dZ"e
dZ#i Z$G dd de%Z&dd Z'G dd d e%Z(G d d d e(Z)G d d d e*Z+G d!d" d"e%Z,G d#d$ d$e%Z-d%d& Z.dS )' )absolute_importdivisionunicode_literals) text_type) http_clienturllibN)BytesIOStringIO) webencodings )EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utilsc C s g | ]}| d qS asciiencode.0item r E/usr/lib/python3.9/site-packages/pip/_vendor/html5lib/_inputstream.py
r c C s g | ]}| d qS r r r r r r r r c C s g | ]}| d qS r r r r r r r r > i
i i i i i i i i i i i i i i i i i i i
i i i i i i i
i
i i i i z[ -
-/:-@\[-`{-~]c @ sH e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dS )BufferedStreamzBuffering for streams that do not have buffering of their own
The buffer is implemented as a list of chunks on the assumption that
joining many strings will be slow since it is O(n**2)
c C s || _ g | _ddg| _d S )Nr r )streambufferposition)selfr" r r r __init__: s zBufferedStream.__init__c C s< d}| j d | jd D ]}|t|7 }q|| jd 7 }|S Nr r )r# r$ len)r% poschunkr r r tell? s
zBufferedStream.tellc C sD |}d}t | j| |k r6|t | j| 8 }|d7 }q||g| _d S r' )r( r# r$ )r% r) offsetir r r seekF s
zBufferedStream.seekc C sT | j s| |S | jd t| j krF| jd t| j d krF| |S | |S d S )Nr r r )r# _readStreamr$ r( _readFromBufferr% bytesr r r readO s
zBufferedStream.readc C s t dd | jD S )Nc S s g | ] }t |qS r )r( r r r r r Y r z1BufferedStream._bufferedBytes..)sumr# r% r r r _bufferedBytesX s z BufferedStream._bufferedBytesc C s< | j |}| j| | jd d7 < t|| jd< |S r' )r" r3 r# appendr$ r( )r% r2 datar r r r/ [ s
zBufferedStream._readStreamc C s |}g }| j d }| j d }|t| jk r|dkr| j| }|t|| kr`|}||| g| _ n"t|| }|t|g| _ |d7 }||||| ||8 }d}q |r|| | d|S )Nr r r )r$ r( r# r7 r/ join)r% r2 ZremainingBytesrvZ bufferIndexZ bufferOffsetZ bufferedDataZ bytesToReadr r r r0 b s$
z BufferedStream._readFromBufferN) __name__
__module__ __qualname____doc__r& r+ r. r3 r6 r/ r0 r r r r r! 3 s r! c K s t | tjs(t | tjjr.t | jtjr.d}n&t| drJt | dt }n
t | t }|rdd |D }|rvt
d| t | fi |S t | fi |S d S )NFr3 r c S s g | ]}| d r|qS ) _encoding)endswith)r xr r r r r z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r)
isinstancer Z HTTPResponser ZresponseZaddbasefphasattrr3 r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargsZ isUnicode encodingsr r r HTMLInputStream} s
rK c @ sp e Zd ZdZdZdd Zdd Zdd Zd d
Zd d Z d
d Z
dddZ dd Z dd Z
d ddZdd ZdS ) rF Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
i ( c C sZ t jsd| _n tddkr$| j| _n| j| _dg| _tddf| _| || _
| dS ) Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
Nu r r utf-8certain) r supports_lone_surrogatesreportCharacterErrorsr( characterErrorsUCS4characterErrorsUCS2ZnewLineslookupEncoding charEncoding
openStream
dataStreamreset)r% rH r r r r& s
z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N r )r* chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacterr5 r r r rX s z HTMLUnicodeInputStream.resetc C s t |dr|}nt|}|S zvProduces a file object from source.
source can be either a file object, local filename or a string.
r3 )rD r r% rH r" r r r rV s
z!HTMLUnicodeInputStream.openStreamc C sT | j }|dd|}| j| }|dd|}|dkr@| j| }n ||d }||fS )N
r r r )r* countr] rfindr^ )r% r, r* ZnLinesZ positionLineZ lastLinePosZpositionColumnr r r _position s
z HTMLUnicodeInputStream._positionc C s | | j\}}|d |fS )z:Returns (line, col) of the current position in the stream.r )re r[ )r% linecolr r r r$ s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return
EOF when EOF is reached.
r )r[ rZ readChunkr r* )r% r[ charr r r ri s
zHTMLUnicodeInputStream.charNc C s |d u r| j }| | j\| _| _d| _d| _d| _| j|}| j rX| j | }d | _ n|s`dS t
|dkrt |d }|dksd| krdkrn n|d | _ |d d }| j r| | |
d d
}|
d d
}|| _t
|| _d S )
NrY r Fr r
i z
rb
T)_defaultChunkSizere rZ r] r^ r* r[ rW r3 r_ r( ordrQ replace)r% rZ r8 Zlastvr r r rh s0
z HTMLUnicodeInputStream.readChunkc C s( t tt|D ]}| jd qd S )Ninvalid-codepoint)ranger( invalid_unicode_refindallr\ r7 )r% r8 _r r r rR s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s d}t |D ]}|rqt| }| }t|||d rrt|||d }|tv rl| j
d d}q|dkr|dkr|t |d kr| j
d qd}| j
d qd S )NF rp Trk i r ) rr finditerrn groupstartr ZisSurrogatePairZsurrogatePairToCodepointnon_bmp_invalid_codepointsr\ r7 r( )r% r8 skipmatchZ codepointr) Zchar_valr r r rS # s" z*HTMLUnicodeInputStream.characterErrorsUCS2Fc C s zt ||f }W nL ty\ ddd |D }|s>d| }td| }t ||f< Y n0 g }|| j| j}|du r| j| jkrqn0| }|| jkr|
| j| j| || _q|
| j| jd | sbqqbd|}|S )z Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
rY c S s g | ]}d t | qS )z\x%02x)rn )r cr r r r H r z5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N) charsUntilRegExKeyErrorr9 recompiler{ r* r[ rZ endr7 rh ) r% Z
charactersZoppositecharsZregexr: mr rr r r
charsUntil: s,
z!HTMLUnicodeInputStream.charsUntilc C s@ |t ur<| jdkr.|| j | _| jd7 _n| jd8 _d S r' )r r[ r* rZ )r% ri r r r ungeti s
z HTMLUnicodeInputStream.unget)N)F)r; r< r= r> rm r& rX rV re r$ ri rh rR rS r r r r r r rF s
&
/rF c @ sL e Zd ZdZdddZdd Zd d
Zdd d Zd
d Zdd Z dd Z
dS )rG rL N windows-1252Tc C s\ | || _t| | j d| _d| _|| _|| _|| _|| _ || _
| || _ |
dS )rM i d N)rV rawStreamrF r& numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrU rX )r% rH r r r r r Z
useChardetr r r r& s z HTMLBinaryInputStream.__init__c C s& | j d j| jd| _t| d S )Nr ro )rU Z
codec_info streamreaderr rW rF rX r5 r r r rX s zHTMLBinaryInputStream.resetc C sJ t |dr|}nt|}z|| W n tyD t|}Y n0 |S r` )rD r r. r+ Exceptionr! ra r r r rV s
z HTMLBinaryInputStream.openStreamc C s | df}|d d ur |S t| jdf}|d d ur:|S t| jdf}|d d urX|S | df}|d d urt|S t| jdf}|d d ur|d jds|S t| jdf}|d d ur|S |r^zddl m
} W n t y Y nv0 g }| }|j s*| j
| j}|sq*|| || q| t|jd }| j
d |d ur^|dfS t| jdf}|d d ur~|S tddfS )NrO r Z tentativezutf-16)UniversalDetectorencodingr ) detectBOMrT r r detectEncodingMetar name
startswithr Z%pip._vendor.chardet.universaldetectorr ImportErrorZdoner r3 r r7 Zfeedcloseresultr. r )r% ZchardetrU r ZbuffersZdetectorr# r r r r r sP
z'HTMLBinaryInputStream.determineEncodingc C s t |}|d u rd S |jdv r(t d}nT|| jd krH| jd df| _n4| jd |df| _| td| jd |f d S )Nutf-16beutf-16lerN r rO z Encoding changed from %s to %s)rT r rU r r. rX r )r% Z newEncodingr r r changeEncoding s
z$HTMLBinaryInputStream.changeEncodingc
C s t jdt jdt jdt jdt jdi}| jd}||dd }d}|sp||}d}|sp||dd }d }|r| j | t
|S | j d
dS dS ) zAttempts to detect at BOM at the start of the stream. If
an encoding can be determined from the BOM return the name of the
encoding otherwise return NonerN r r zutf-32lezutf-32be N ru r ) codecsBOM_UTF8 BOM_UTF16_LE BOM_UTF16_BE BOM_UTF32_LE BOM_UTF32_BEr r3 getr. rT )r% ZbomDictstringr r. r r r r s$
z HTMLBinaryInputStream.detectBOMc C sH | j | j}t|}| j d | }|durD|jdv rDtd}|S )z9Report the encoding declared by the meta element
r Nr rN )r r3 r EncodingParserr. getEncodingr rT )r% r# parserr r r r r 3 s z(HTMLBinaryInputStream.detectEncodingMeta)NNNNr T)T) r; r< r= r> r& rX rV r r r r r r r r rG z s
*
>"rG c @ s e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z e e e
Z
dd Ze eZefddZdd Zdd Zdd Zd S )
EncodingByteszString-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raisedc C s t | | S N)r2 __new__lowerr% valuer r r r F s zEncodingBytes.__new__c C s
d| _ d S )Nr )re r r r r r& J s zEncodingBytes.__init__c C s | S r r r5 r r r __iter__N s zEncodingBytes.__iter__c C s> | j d }| _ |t| kr"tn |dk r.t| ||d S )Nr r re r(
StopIterationrE r% pr r r __next__Q s zEncodingBytes.__next__c C s | S r )r r5 r r r nextY s zEncodingBytes.nextc C sB | j }|t| krtn |dk r$t|d | _ }| ||d S r' r r r r r previous] s zEncodingBytes.previousc C s | j t| krt|| _ d S r re r( r )r% r$ r r r setPositionf s zEncodingBytes.setPositionc C s* | j t| krt| j dkr"| j S d S d S )Nr r r5 r r r getPositionk s
zEncodingBytes.getPositionc C s | | j | j d S Nr )r$ r5 r r r getCurrentByteu s z EncodingBytes.getCurrentBytec C sH | j }|t| k r>| ||d }||vr4|| _|S |d7 }q|| _dS )z Skip past a list of charactersr Nr$ r( re r% r r r| r r r rz z s
zEncodingBytes.skipc C sH | j }|t| k r>| ||d }||v r4|| _|S |d7 }q|| _d S r r r r r r skipUntil s
zEncodingBytes.skipUntilc C s( | || j}|r$| jt|7 _|S )zLook for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone)r r$ r( )r% r2 r: r r r
matchBytes s zEncodingBytes.matchBytesc C s< z | || jt| d | _W n ty6 tY n0 dS )zLook for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the matchr T)indexr$ r( re
ValueErrorr r1 r r r jumpTo s
zEncodingBytes.jumpToN)r; r< r= r> r r& r r r r r r propertyr$ r currentBytespaceCharactersBytesrz r r r r r r r r B s
r c @ sX e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z dd Z dS )r z?Mini parser for detecting character encoding from meta elementsc C s t || _d| _dS )z3string - the data to work on for encoding detectionN)r r8 r r% r8 r r r r& s
zEncodingParser.__init__c
C s d| j vrd S d| jfd| jfd| jfd| jfd| jfd| jff}| j D ]}d}z| j d W n tyx Y qY n0 |D ]B\}}| j |r~z| }W qW q~ ty d}Y qY q~0 q~|sH qqH| j S ) Ns r8 r r5 r r r r s z EncodingParser.handleCommentc C s | j jtvrdS d}d }| }|d u r,dS |d dkr\|d dk}|r|d ur|| _dS q|d dkr|d }t|}|d ur|| _dS q|d dkrtt|d }| }|d urt|}|d ur|r|| _dS |}qd S ) NTFr s
http-equivr s content-type charsets content) r8 r r getAttributer rT ContentAttrParserr parse)r% Z hasPragmaZpendingEncodingattrZtentativeEncodingcodecZ
contentParserr r r r s8 zEncodingParser.handleMetac C s
| dS )NF)handlePossibleTagr5 r r r r s z%EncodingParser.handlePossibleStartTagc C s t | j | dS )NT)r r8 r r5 r r r r s
z#EncodingParser.handlePossibleEndTagc C sb | j }|jtvr(|r$| | dS |t}|dkrD| n| }|d ur^| }qLdS )NTr )r8 r asciiLettersBytesr r r spacesAngleBracketsr )r% ZendTagr8 r| r r r r r s
z EncodingParser.handlePossibleTagc C s | j dS )Nr r r5 r r r r s zEncodingParser.handleOtherc C s | j }|ttdgB }|dv r&dS g }g }|dkr>|r>qnX|tv rR| }qnD|dv rhd|dfS |tv r|| n|du rdS || t|}q.|dkr| d|dfS t| | }|dv r2|}t|}||kr
t| d|d|fS |tv r$|| q|| qnJ|dkrJd|dfS |tv rd|| n|du rrdS || t|}|t
v rd|d|fS |tv r|| n|du rdS || q|dS ) z_Return a name,value pair for the next attribute in the stream,
if one is found, or None /)r NN =)r r r ) ' "r ) r8 rz r frozensetr9 asciiUppercaseBytesr7 r r r r )r% r8 r| ZattrNameZ attrValueZ quoteCharr r r r s`
zEncodingParser.getAttributeN)
r; r< r= r> r& r r r r r r r r r r r r r s $r c @ s e Zd Zdd Zdd ZdS )r c C s
|| _ d S r )r8 r r r r r& a s zContentAttrParser.__init__c C s z| j d | j jd7 _| j | j jdks sF
J g Ib ='