- z d dl mZmZmZ d dlmZ d dlmZ d dlm Z
m Z e
e_ e e_ dgZ
G d de Z G d de Z G d d
e Zd S ) )absolute_importdivisionunicode_literalsstr)urllib)parserequestRobotFileParserc J e Zd ZdZd
dZd Zd Zd Zd Zd Z d Z
d
Z d Z d S )r zs This class provides a set of methods to read, parse and answer
questions about a single robots.txt file.
c v g | _ d | _ d| _ d| _ | | d| _ d S )NFr )entries
default_entry disallow_all allow_allset_url last_checkedselfurls u/builddir/build/BUILD/cloudlinux-venv-1.0.10/venv/lib/python3.11/site-packages/future/backports/urllib/robotparser.py__init__zRobotFileParser.__init__ s@ ! ! S c | j S )zReturns the time the robots.txt file was last fetched.
This is useful for long-running web spiders that need to
check for new robots.txt files periodically.
)r r s r mtimezRobotFileParser.mtime& s
r c @ ddl }| | _ dS )zYSets the time the robots.txt file was last fetched to the
current time.
r N)timer )r r s r modifiedzRobotFileParser.modified/ s#
IIKKr c | || _ t j | dd \ | _ | _ dS )z,Sets the URL referring to a robots.txt file. N)r r r urlparsehostpathr s r r zRobotFileParser.set_url7 s4 %| 4 4S 9 9!A# > 4999r c t j | j }| }| | d dS # t j j $ r:}|j
dv rd| _ n |j
dk r
d| _ Y d}~dS Y d}~dS Y d}~dS d}~ww xY w)z4Reads the robots.txt URL and feeds it to the parser.zutf-8)i i Ti N)
r r
urlopenr readr decode
splitlineserror HTTPErrorcoder r )r frawerrs r r) zRobotFileParser.read< s 9&&tx00A &&((C JJszz'**5577 8 8 8 8 8 |% & & &x:%%$(!!S !% ! "!!!!! &s $A6 6C
#B??Cc p d|j v r| j || _ d S d S | j | d S N*)
useragentsr r append)r entrys r
_add_entryzRobotFileParser._add_entryI sM %" " "!)%*""" *)
L & & & & &r c R d}t }|D ]}|sB|dk rt }d}n+|dk r%| | t }d}| d }|dk r
|d| }| }|s| dd }t | dk rH|d |d< t j |d |d< |d dk rM|dk r#| | t }|j
|d d}o|d dk r8|dk r0|j t |d d d}|d d
k r6|dk r0|j t |d d d}|dk r| | dS dS ) zParse the input lines from a robots.txt file.
We allow that a user-agent: line is not preceded by
one or more blank lines.
r r" #N:z
user-agentdisallowFallowT)Entryr8 findstripsplitlenlowerr r unquoter5 r6 rulelinesRuleLine)r linesstater7 lineis r r zRobotFileParser.parseR s ! " ! "D
A:: !GGE EEaZZOOE*** !GGE E #AAvvBQBx::< 4 \**6<+?+?+D+DEE
l%%r"Z_ j .
0C'E F Fl %% C\ , ,E **
, s+++++
, 5%//44 4tr c J d d | j D S )Nr
c 2 g | ]}t | d z S )
r ).0r7 s r
z+RobotFileParser.__str__.. s# DDDeE
T)DDDr )joinr r s r __str__zRobotFileParser.__str__ s% wwDDt|DDDEEEr N)r
)
__name__
__module__ __qualname____doc__r r r r r) r8 r rW r^ r r r r s
! ! !( ( (? ? ?
9 9 9' ' '0# 0# 0#f .F F F F Fr c $ e Zd ZdZd Zd Zd ZdS )rG zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c p |dk r|sd}t j | | _ || _ d S )Nr
T)r r rR r& rT )r r& rT s r r zRuleLine.__init__ s6 2::i: IL&&t,, "r c L | j dk p| | j S r3 )r&
startswith)r filenames r rS zRuleLine.applies_to s$ yC A8#6#6ty#A#AAr c . | j rdpddz | j z S )NAllowDisallowz: )rT r& r s r r^ zRuleLine.__str__ s *78jD@49LLr N)r_ r` ra rb r rS r^ rc r r rG rG sS 1 1# # #B B BM M M M Mr rG c * e Zd ZdZd Zd Zd Zd ZdS )r? z?An entry has one or more user-agents and zero or more rulelinesc " g | _ g | _ d S )N)r5 rF r s r r zEntry.__init__ s r c g }| j D ]}| d|dg | j D ]&}| t | dg 'd | S )Nz User-agent: rZ r
)r5 extendrF r r] )r retagentrJ s r r^ z
Entry.__str__ su _ 6 6E JJt4 5 5 5 5N * *D JJD 4( ) ) ) )wws||r c | d d }| j D ]&}|dk r dS | }||v r dS 'dS )z2check if this entry applies to the specified agentrM r r4 TF)rB rD r5 )r rU rq s r rS zEntry.applies_to sp OOC((+1133 _ E||ttKKMME !!tt "ur c V | j D ] }| | r |j c S !dS )zZPreconditions:
- our agent applies to this entry
- filename is URL decodedT)rF rS rT )r rh rJ s r rT zEntry.allowance sA N & &Dx((
&~%%%
&tr N)r_ r` ra rb r r^ rS rT rc r r r? r? sV II r r? N)
__future__r r r future.builtinsr future.backportsr future.backports.urllibr _parser
_request__all__objectr rG r? rc r r r| s B B B B B B B B B B
$ # # # # # H H H H H H H H
EF EF EF EF EFf EF EF EFPM M M M Mv M M M"" " " " "F " " " " "r