a
!q ^a @ s d Z ddlZddlZddlZddlmZ ddlm Z ddl
m Z m Z m
Z
mZmZ ddlmZ ddlmZmZ ddl
mZ dd lmZmZmZmZmZm Z m Z m Z m Z m Z m!Z!m"Z"m#Z# d
Z$G d d d ej%j&Z'de(e(e
e) e(dddZ*dS )z>html2text: Turn HTML into equivalent Markdown-structured text. N)wrap)DictListOptionalTupleUnion )config)
AnchorElement ListElement) OutCallback)
dumb_css_parser
element_style escape_mdescape_md_sectiongoogle_fixed_width_fontgoogle_has_heightgoogle_list_stylegoogle_text_emphasishnlist_numbering_startpad_tables_in_textskipwrap unifiable_n)i r c s e Zd Zddejfee eedd fdd
Z edd fdd Z
eedd d
Z edd d d
Z edddZ
eddddZeddddZeeeeee f ddddZeddddZeeee f ee d d d Zeeeef eeef dd d d!Zeeeee f edd"d#d$Zddd%d&Zddd'd(Zddd)d*Zd=eee eef dd,d-d.Z d>eedd/d0d1Z eed2d3d4Z eedd5d6Z eeef ed7d8d9Z!eed:d;d<Z" Z#S )? HTML2TextN )outbaseurl bodywidthreturnc s
t jdd d| _d| _d| _tj| _tj| _ tj
| _ || _ tj
| _tj| _tj| _tj| _tj| _tj| _tj| _tj| _ tj | _ tj | _ tj!| _"tj#| _$d| _%d| _&d| _'d| _(tj)| _*tj+| _,d| _-tj.| _/tj0| _1tj2| _3tj4| _5tj6| _7d| _8tj9| _:tj;| _<|du r | j=| _>n|| _>g | _?d| _@d| _Ad| _Bd| _Cd| _Dg | _Eg | _Fd| _Gd| _HtIJd | _Kd| _Lg | _Md| _Nd| _Od| _Pd| _Qd| _Rd
| _Sd| _Td| _Ud| _Vi | _Wg | _Xd| _Yd| _Zd| _[d| _\d| _]i | _^|| __d| _`d| _ad
| _bd
| _cd tjdd < dS )
z
Input parameters:
out: possible custom replacement for self.outtextf (which
appends lines of text).
baseurl: base URL of the document we process
F)Zconvert_charrefsr *_z**NTz^[a-zA-Z+]+://r _place_holder;nbsp)esuper__init__
split_next_tdtd_count table_startr Z UNICODE_SNOB unicode_snobZ ESCAPE_SNOB escape_snobZLINKS_EACH_PARAGRAPHlinks_each_paragraph
body_widthZSKIP_INTERNAL_LINKSskip_internal_linksZ INLINE_LINKS inline_linksZ
PROTECT_LINKS
protect_linksZGOOGLE_LIST_INDENTgoogle_list_indentZIGNORE_ANCHORS ignore_linksZ
IGNORE_IMAGES
ignore_imagesZIMAGES_AS_HTMLimages_as_htmlZ
IMAGES_TO_ALT
images_to_altZIMAGES_WITH_SIZEimages_with_sizeZIGNORE_EMPHASISignore_emphasisZ
BYPASS_TABLES
bypass_tablesZ
IGNORE_TABLES
ignore_tables
google_doc ul_item_mark
emphasis_mark strong_markZSINGLE_LINE_BREAKsingle_line_breakZUSE_AUTOMATIC_LINKSuse_automatic_linkshide_strikethroughZ MARK_CODE mark_codeZWRAP_LIST_ITEMSwrap_list_itemsZ
WRAP_LINKS
wrap_linksZ
PAD_TABLES
pad_tablesZDEFAULT_IMAGE_ALTdefault_image_alt tag_callbackZ
OPEN_QUOTE
open_quoteZ CLOSE_QUOTE close_quoteouttextfr outtextlistquietp_poutcountstartspaceaastackmaybe_automatic_link
empty_linkrecompileabsolute_url_matcheracountlist
blockquoteprestartprecodequote br_toggle lastWasNL lastWasListstyle style_def tag_stackemphasisdrop_white_spaceinheader
abbr_title abbr_data abbr_listr stressedpreceding_stressedpreceding_data current_tag UNIFIABLE)selfr r r __class__ 6/usr/lib/python3.9/site-packages/html2text/__init__.pyr& % s
zHTML2Text.__init__)datar c s | dd}t | d S )Nzz )replacer% feed)ro rt rp rr rs rv s zHTML2Text.feedc C s8 | | | d | | }| jr0t|S |S d S )Nr )rv optwrapfinishrD r )ro rt Zmarkdownrr rr rs handle s
zHTML2Text.handle)sr c C s" | j | |r |d dk| _d S )N
)rJ appendr_ )ro rz rr rr rs rI s zHTML2Text.outtextf)r c C sX | | | jddd d| j}| jr>tjjd }nd}| d|}g | _|S )Nr endforceznbsp; r# )
closepbrojoinrJ r* htmlentitieshtml5ru )ro Zouttextr$ rr rr rs rx s zHTML2Text.finish)cr c C s | | |d d S NT) handle_datacharref)ro r rr rr rs handle_charref s zHTML2Text.handle_charrefc C s | |}|r| |d d S r ) entityrefr )ro r refrr rr rs handle_entityref s
zHTML2Text.handle_entityref)tagattrsr c C s | j |t|dd d S )NTrN )
handle_tagdict)ro r r rr rr rs handle_starttag s zHTML2Text.handle_starttag)r r c C s | j |i dd d S )NFr )r )ro r rr rr rs
handle_endtag s zHTML2Text.handle_endtag)r r c C s d|vr dS d}t | jD ]p\}}d|jv r~|jd |d kr~d|jv sPd|v rzd|jv r~d|v r~|jd |d kr~d}nd}|r| S qdS )z
:type attrs: dict
:returns: The index of certain set of attributes (of a link) in the
self.a list. If the set of attributes is not found, returns None
:rtype: int
hrefNFtitleT) enumeraterP r )ro r matchirP rr rr rs
previousIndex s"
zHTML2Text.previousIndex)rN tag_style parent_styler c C s t |}t |}d|v o | j}d}tjD ] }||v o:||v}|r( qFq(d|v oTd|v} t|ont| on| j }
|r|s| s|
r| jd7 _|r| jd7 _| r| | j | j
d7 _
|r| | j | j
d7 _
|
r| d | j
d7 _
d| _ n|s| s|
r*| jd8 _d| _
|
rX| j
rH| j
d8 _
n
| d d| _ |r| j
rv| j
d8 _
n | | j | r| j
r| j
d8 _
n | | j |s| r| js| d |r| jd8 _dS ) z/
Handles various text emphases
z line-throughFitalicr `Tr N)r r@ r ZBOLD_TEXT_STYLE_VALUESr rZ rd rK r r< re r= r\ rO ) ro rN r r Z tag_emphasisZparent_emphasisZ
strikethroughZboldZ bold_markerr fixedrr rr rs handle_emphasis sb
zHTML2Text.handle_emphasis)r r rN r c C s || _ | jd ur(| | |||du r(d S |rb| jd urb|dvrb|dksL| jrb| d d | _d| _| jri }|r| jr| jd d }t|| j |}| j
|||f n4| jr| j nd i i f\}}}| jr| jd d }t |r|
|rd| _| t |d d n
d| _d S |d
v rl| jrP|rFt|rF|
n| n | jrd|d krdn|
|d kr|r| jd
kr| d n
| d |dkr|r|
| d |
|dv r|r| jd7 _n| jd8 _|dkr$|r| jd7 _n| jd8 _|dv r4d
| _|dkr|rp|
| jddd d| _| jd7 _n| jd8 _|
ttddd}|d v r| js|r|| rd | j }n| j}| | |rd| _|d v r*| js*|r|| rd | j } n| j} | | |r*d| _|d v rd|rJ|| rJd }
nd }
| |
|rdd| _| jr| js| ||| |d!v r| j s| d" | j | _ |d#kr |rd | _ d$| _ d%|v r |d% | _ n&| j d ur| j | j!| j < d | _ d | _ |d&kr6| j"s | | j# n | | j$ | j" | _"dVtt%t%d d'd(d)} |d*kr| j&s|rd+|v r|d+ d ur| j'r|d+ (ds| j
| |d+ | _d| _| j)rd,|d+ d- |d+< n | j
d n| jr| j } | jr| jsd | _n| r| jr*| d d| _d | _| j*r\| +d%p@d$}
t,|
}
| | | d+ |
nb| -| }|d ur|| j.| }n*| j/d7 _/t0| | j/| j1}| j.
| | d.t%|j2 d/ |dkr|r| jsd0|v r| j3s|d0 |d+< |+d1p| j4}| j5s*| j6rd2|v s*d3|v r| d4|d0 d5 d2|v r`| d6|d2 d5 d3|v r| d7|d3 d5 |r| d8| d5 | d9 d S | jd ur| j}| j3rt,||kr| j78|r| d,t,| d- d| _d S | d d | _d| _| j3r*| t,| n| d:t,| d/ | j*rz|+d+pVd$}| d;t,t9:| j;| d< nb| -|}|d ur| j.| }n*| j/d7 _/t0|| j/| j1}| j.
| | dt%|j2 d/ |d=kr|r|
|d>kr |s | < |d?kr&|r&| d@ |d?kr>|s>| < |dAv r| j=s`| j>s`|
|r| jrxt?|}n|}t@|}| j=
tA|| n,| j=r| j= | js| j=s| dB d| _>nd| _>|dCk r| < | r| j=r| j=d }n
tAdDd
}| j r | B|}n
tC| j=}| dE| |jDdDk rR| | jEd n.|jDdFk r| jFd7 _F| t%|jFdG d| _|dHv rJ| jG r|dIk rJ| rn| n n| jH
r.| r| |dJv
r| r| dKI| n| dLI| n(|
r| dMI| n| dNI| n |dOk
r|
rld| _J| jK
r| d,tLjM d- | d n&| jK
r| dPtLjM d- | d |dJv
r|
r| jN
r| dQ d| _N|dIk
r|
rd
| _O|dIk
r|
sd| _N| |dIk r,| s,| jJ r,| dRPdSg| jO | d| _J|dJv rJ| rJ| jOd7 _O|dTk r| rhd| _Qd| _ nd| _ | jR r| SdU |
d S )WNT)pdivra dldtZimg[Fr{ #r )r r r brr z
>
hrz* * *)headra Zscriptr ra )bodyrY > r )ro r c S s t | jotd| jd S )Nz[^\s]r{ )boolrl rT r ro rr rr rs no_preceding_space s z0HTML2Text.handle_tag..no_preceding_space)Zemr u)strongb)delstrikerz z ~~z~~)Zkbdr\ ttr abbrr r q)ro linkr r c S s@ t | j|}| r d|nd}| djt||d d S )Nz "{}"r z]({url}{title}))urlr )urlparseurljoinr stripformatr r )ro r r r rr rr rs link_url s z&HTML2Text.handle_tag..link_urlrP r <>z][]srcaltwidthZheightz
![z/>z![()r r dd )olulr| lir r z. )tabletrtdthr )r r z<{}>
z
</{}>z<{}>z</{}>r z</z| |z---rZ z
[/code])r )Trm rF rR r3 r rS r: rc r rb r} popr r rf r soft_brrQ rY rK ra rN r r r7 r< rj r= r rZ r\ rg rh ri r] rG rH strr2 r.
startswithr0 r/ getr r rP rW r
rM countr5 rE r4 r6 rV r r r r r rX r` r r r google_nest_countlennamer; Znumr9 r8 r r) rD r ZTABLE_MARKER_FOR_PADr](z)
zHTML2Text.handle_tagc C s | j dkrd| _ dS )z Pretty print has a line breakr r N)rL r rr rr rs r s
z
HTML2Text.pbrc C s | j r
dnd| _dS )z Set pretty print to 1 or 2 linesr r N)r> rL r rr rr rs r s z HTML2Text.pc C s | d| _dS )z Soft breaksr N)r r^ r rr rr rs r s zHTML2Text.soft_brF)rt puredatar r c
C s | j dur| j |7 _ | js| jrR| }| jrD| jsD| jsD|}|dkrRd| _|r| jstdd|}|r|d dkrd| _ |dd }|s|sdS | j
r| ds| d sd| }| j r|
d
d| _d | j }|r|r|d d ks| jr|d7 }| jr6| js|d 7 }|d t| j 7 }|dd| }| j
rVd
| _
| jrV|d}| jrpd
| _ d| _d
| _|dkrd| _|
d d
| _ | jr|
| jd | | j d
| _ d| _| j r| js|
d d
| _ | jr| jdkr| js|dkr|dkr|
d g }| jD ]z}| j|jkr|
dt|j d t | j |j d d|j v r|
d|j d d |
d n
| | q"| j|kr|
d || _| j r|dkr| j ! D ]$\}} |
d| d | d qd| _|
| | jd7 _dS )z6
Deal with indentation and whitespace
Nr r z\s+r Tr r| z
z
[code]r r Fr~ r z [z]: r r z (r z *[)"rh rK r: lstripre rZ r\ rT subrO r[ r rA r rL rY rX r ru rN r^ r_ rP r, rM r r r r r r r} ri items)
ro rt r r Zlstripped_dataZbqZnewar r Z
definitionrr rr rs r s
z HTML2Text.o)rt entity_charr c C s |sd S | j r$| }d| _ d| _n8| jr\td|d rVt| jsV| jdvrVd| }d| _| jrr| j t
| | j d ur| j }||kr| j |r| j
r| d| d d| _d S | d d | _ d| _| js| js|st|| jd
}|| _| j|dd d S ) NFTz[^\s.!?]r )rP r\ rZ r r r r )Zsnob)r )rj r rk rT r r rm ra rb updater
rR rV r? r rS r\ rZ r r+ rl )ro rt r r rr rr rs r ! sF
zHTML2Text.handle_data)r r c C sb |d dv r t |dd d}nt |}| js>|tv r>t| S z
t|W S ty\ Y dS 0 d S )Nr )xXr r r )intr* r chr
ValueError)ro r r rr rr rs r L s
zHTML2Text.charrefc C sd | j s|tjv rtj| S ztjj|d }W n tyL d| d Y S 0 |dkr`tj| S |S )N;&r$ )r* r rn r r r KeyError)ro r Zchrr rr rs r Z s
zHTML2Text.entityref)ra r c C s* d}d|v r&t |d dd | j }|S )zq
Calculate the nesting count of google doc lists
:type style: dict
:rtype: int
r z margin-leftN)r r1 )ro ra r rr rr rs r c s zHTML2Text.google_nest_count)textr c C s | j s
|S d}d}| js d| _|dD ]}t|dkrt|| j| jsd}|d| j rbd}n|drpd}t || j d|d}|d
|7 }| dr|d 7 }d
}q|r|d7 }d
}q|d 7 }d }qt j
|s||d 7 }d
}q(|d k r(|d7 }|d
7 }q(|S )
zi
Wrap all paragraphs in the provided text.
:type text: str
:rtype: str
r r Fr| r r r )Zbreak_long_wordssubsequent_indentr r z
r )r- rC r/ splitr r rB r r; r r endswithr ZRE_SPACEr )ro r resultnewlinesZparaindentwrappedrr rr rs rw q sH
zHTML2Text.optwrap)FF)F)$__name__
__module__ __qualname__r
BODY_WIDTHr r r r r& rv ry rI rx r r r r r r r r r r r r r r r r r r r r rw
__classcell__rr rr rp rs r $ sN d " J { q+ r r )r r r r c C s$ |d u rt j}t||d}|| S )N)r r )r r r ry )r r r hrr rr rs html2text s r )r N)+__doc__Z
html.entitiesr Z html.parserrT urllib.parseparser textwrapr typingr r r r r r r elementsr
r r Zutilsr
r r r r r r r r r r r r __version__parserZ
HTMLParserr r r r rr rr rr rs s( <