Oՠp d dl Z d dlZd dlZdgZ e j dd Z G d d Z G d d Z G d d Z dS )
NRobotFileParser RequestRatezrequests secondsc Z e Zd Z ddZd Zd Zd Zd Zd Zd Z d Z
d
Z d Z d Z
d
ZdS )r c g | _ g | _ d | _ d| _ d| _ | | d| _ d S )NFr )entriessitemaps
default_entry disallow_all allow_allset_url last_checkedselfurls 9/opt/alt/python311/lib64/python3.11/urllib/robotparser.py__init__zRobotFileParser.__init__ sG
! ! S c | j S N)r r s r mtimezRobotFileParser.mtime% s r c B dd l }| | _ d S )Nr )timer )r r s r modifiedzRobotFileParser.modified. s( IIKKr c ~ || _ t j | dd \ | _ | _ d S )N )r urllibparseurlparsehostpathr s r r
zRobotFileParser.set_url6 s7 : %| 4 4S 9 9!A# > 4999r c t j | j }| }| | d d S # t j j $ rK}|j
dv rd| _ n)|j
dk r|j
dk rd| _ Y d }~d S Y d }~d S Y d }~d S Y d }~d S d }~ww xY w)Nzutf-8)i i Ti i )
r requesturlopenr readr decode
splitlineserror HTTPErrorcoder r )r frawerrs r r' zRobotFileParser.read; s B 9&&tx00A &&((C JJszz'**5577 8 8 8 8 8 |% & & &x:%%$(!!S SX^^!% ! "!!!!!%3^^^^^ &s $A7 7C .CCc p d|j v r| j || _ d S d S | j | d S N*)
useragentsr
r append)r entrys r
_add_entryzRobotFileParser._add_entryH sM %" " "!)%*""" *)
L & & & & &r c > d}t }| |D ]V}|sB|dk rt }d}n+|dk r%| | t }d}| d }|dk r
|d | }| }|s| dd }t
| dk r|d |d< t j
|d |d< |d dk rM|dk r#| | t }|j |d d}o|d dk r8|dk r0|j
t |d d d}|d d k r8|dk r0|j
t |d d
d}|d d k rP|dk rH|d rt! |d |_ d}S|d d k r|dk r|d d
}t
| dk r|d rg|d r;t% t! |d t! |d |_ d}*|d dk r | j |d X|dk r| | d S d S )Nr r #:z
user-agentdisallowFallowTz crawl-delayz request-rate/sitemap)Entryr r6 findstripsplitlenlowerr r unquoter3 r4 rulelinesRuleLineisdigitintdelayr req_rater )r linesstater5 lineinumberss r r zRobotFileParser.parseQ sO
7 2 7 2D
A:: !GGE EEaZZOOE*** !GGE E #AAvvBQBx::< 4
5 \**6<+?+?+D+DEE
l%%r"Z_ j .
0C'E F Fl %% C\ , ,E **
, s+++++
, 5%//44 4tr c | sd S | j D ] }| | r |j c S !| j r | j j S d S r )r r rW rJ r
r rY r5 s r crawl_delayzRobotFileParser.crawl_delay sm zz|| 4\ # #E **
# {"""
# ,%+ +tr c | sd S | j D ] }| | r |j c S !| j r | j j S d S r )r r rW rK r
r] s r request_ratez RobotFileParser.request_rate sm zz|| 4\ & &E **
& ~%%%
& /%. .tr c " | j sd S | j S r )r r s r site_mapszRobotFileParser.site_maps s } 4} r c | j }| j || j gz }d t t | S )Nz
)r r
joinmapstr)r r s r __str__zRobotFileParser.__str__ s> , ) !3 44G{{3sG,,---r N)r )__name__
__module__ __qualname__r r r r
r' r6 r r[ r^ r` rb rg r r r r s
! ! !( ( (? ? ?
9 9 9' ' 'G# G# G#R :
. . . . .r c " e Zd Z d Zd Zd ZdS )rG c |dk r|sd}t j t j | }t j | | _ || _ d S )Nr T)r r rR r! rV r# rX )r r# rX s r r zRuleLine.__init__ s[ 2::i: I|&&v|'<'9zTADIMMr N)rh ri rj r rW rg rk r r rG rG sN 1# # #B B BN N N N Nr rG c ( e Zd Z d Zd Zd Zd ZdS )r? c > g | _ g | _ d | _ d | _ d S r )r3 rF rJ rK r s r r zEntry.__init__ s"
r c | g }| j D ]}| d| | j | d| j | j ,| j }| d|j d|j | t t | j d
| S )Nz User-agent: z
Crawl-delay: zRequest-rate: r=
) r3 r4 rJ rK requestssecondsextendre rf rF rd )r retagentrates r rg z
Entry.__str__ s _ / /E JJ-e-- . . . . : ! JJ3tz33 4 4 4 = $=D JJF
FF FF G G G
3sDN++,,,yy~~ r c | d d }| j D ]&}|dk r dS | }||v r dS 'dS )Nr= r r2 TF)rB rD r3 )r rY r| s r rW zEntry.applies_to sq @ OOC((+1133 _ E||ttKKMME !!tt "ur c X | j D ] }| | r |j c S !dS )NT)rF rW rX )r rp rN s r rX zEntry.allowance
sF % N & &Dx((
&~%%%
&tr N)rh ri rj r rg rW rX rk r r r? r? sS I
r r? )
collections urllib.parser urllib.request__all__
namedtupler r rG r? rk r r r s
$k$]4FGG ~. ~. ~. ~. ~. ~. ~. ~.BN N N N N N N N$( ( ( ( ( ( ( ( ( (r