3
ՔRe; @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl m
Z
yddl m Z W n e
k
r ddlm Z Y nX ddlmZ ddlmZmZmZmZ ddlZddlmZmZmZmZm Z m Z m Z m Z m Z m!Z!m"Z"m#Z#m$Z$ ddlm%Z% dd l&m'Z' dd
l(m)Z) dd l*m+Z+ dd l,m-Z- dd
l.m/Z/ ej0dZ1ej0dej2Z3ej0dZ4ej0dej2j5Z6dj7 Z8ddddgZ9dZ:dZ;e;jdd Z?d d Z@d d ZAdGd dZBdHd!d"ZCdId#d$ZDde dfd%dZEdJd&d'ZFd(d) ZGej0d*ej2ZHeGd+d, ZIG d-d. d.eJZKG d/d0 d0eKZLG d1d de ZMej0d2jNZOd3d4 ZPd5d6 ZQd7d8 ZRdKd9d:ZSd;d< ZTG d=d> d>eJZUG d?d@ d@ejVZWejXjYfdAdBZZeSe:eZZZdCdD Z[dEdF Z\dS )Lz#PyPI and direct package downloading N)wraps) splituser)six)urllib http_client configparsermap)
CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST) ssl_support)log)DistutilsError) translate)get_all_headers)Wheelz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)
\s+\(md5\)z([-+.a-z0-9]{2,}):z .tar.gz .tar.bz2 .tar .zip .tgz PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_name z s z(interpret_distro_name..r; Nr9 )
py_versionrK r\ )rA anyrangerY r
join)rS rZ rP re rK r\ rC rc r& r& r' r s
$c c sn t }|j}|dkr>xTtjj|j| D ]}|| |V q&W n,x*| D ]"}||}||krD|| |V qDW dS )zHList unique elements, preserving order. Remember all elements ever seen.N)setaddr moves filterfalse __contains__)iterablekeyseenZseen_addelementkr& r& r' unique_everseen s
rs c s t fdd}|S )zs
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | |S )N)rs )argskwargs)funcr& r' wrapper s z unique_values..wrapper)r )rv rw r& )rv r'
unique_values s rx z(<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c c s xvt j|D ]h}|j \}}tttj|j jd}d|ksFd|kr x,t j|D ] }t
j j | t
|jdV qRW q W xHdD ]@}|j|}|d kr~t j||}|r~t
j j | t
|jdV q~W dS )
zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager: r9
| Home Page | Download URLN)rz r{ r= )RELfinditergroupsri r strstripr1 rA HREFr r# urljoin
htmldecoderO findsearch)rB pagerN tagrelZrelsposr& r& r' find_external_links s "
r c @ s( e Zd ZdZdd Zdd Zdd ZdS ) ContentCheckerzP
A null content checker that defines the interface for checking content
c C s dS )z3
Feed a block of data to the hash.
Nr& )selfblockr& r& r' feed s zContentChecker.feedc C s dS )zC
Check the hash. Return False if validation fails.
Tr& )r r& r& r' is_valid s zContentChecker.is_validc C s dS )zu
Call reporter with information about the checker (hash name)
substituted into the template.
Nr& )r reportertemplater& r& r' report s zContentChecker.reportN)__name__
__module__ __qualname____doc__r r r r& r& r& r' r s r c @ sB e Zd ZejdZdd Zedd Zdd Z dd Z
d
d Z d S )
HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s || _ tj|| _|| _d S )N) hash_namehashlibnewhashexpected)r r r r& r& r' __init__ s zHashChecker.__init__c C s> t jj|d }|st S | jj|}|s0t S | f |j S )z5Construct a (possibly null) ContentChecker from a URLr9 r= )r r# r? r patternr groupdict)clsrB rI rN r& r& r' from_url s zHashChecker.from_urlc C s | j j| d S )N)r update)r r r& r& r' r ! s zHashChecker.feedc C s | j j | jkS )N)r hexdigestr )r r& r& r' r $ s zHashChecker.is_validc C s || j }||S )N)r )r r r msgr& r& r' r ' s
zHashChecker.reportN) r r r ra compiler r classmethodr r r r r& r& r& r' r s r c s< e Zd ZdZdKddZdLd d
ZdMd d ZdNd
dZdd Zdd Z dd Z
dd Z dOddZ dd Z
dP fdd Zd d Zd d Zd!d" Zd#d$ Zd%d& ZdQd'd(ZdRd)d*Zd+d, Zd-Zd.d/ Zd0d1 ZdSd2d3Zd4d5 Zd6d7 Z d8d9 Z d:d; Z dd?Z!d@dA Z"dBdC Z#dDdE Z$dFdG Z%dHdI Z& Z'S )Ur z;A distribution index that scans web pages for download URLs https://pypi.python.org/simple*NTc O s t j| f|| |dd |jd | _i | _i | _i | _tjdj t
t |j | _
g | _|ortjor|prtj }|rtj|| _n
tjj| _d S )Nr8 |)r r r2 index_url scanned_urls fetched_urls
package_pagesra r rh r r rN allowsto_scanr Z is_availableZfind_ca_bundleZ
opener_foropenerr requesturlopen)r r hostsZ ca_bundleZ
verify_sslrt kwZuse_sslr& r& r' r / s zPackageIndex.__init__Fc C s || j kr| rdS d| j |< t|s4| j| dS tt|}|r^| j|sRdS | jd| |sr| sr|| jkrtt| j | dS | j|sd| j|< dS | j
d| d| j|< d}| j ||| }|dkrdS d| j|j < d|j
jddj kr|j dS |j }|j }t|tsRt|tjjr4d }n|j
jd
pDd }|j|d }|j x6tj|D ](} tjj |t | j d }
| j |
qfW |j | j!rt"|d
ddkr| j#||}dS )z.)filterr^ rF r r itertoolsstarmap
scan_egg_link)r search_pathdirsZ egg_linksr& r& r' scan_egg_links s zPackageIndex.scan_egg_linksc
C s t tjj|| }ttd ttj|}W d Q R X t |dkrDd S |\}}x>t
tjj||D ](}tjj|f| |_ t |_
| j| q`W d S )Nr; )openr^ rF rh r r r r r rY r rS r
rK rj )r rF r Z raw_lineslinesZegg_pathZ
setup_pathrQ r& r& r' r s zPackageIndex.scan_egg_linkc
s fdd}xHt j|D ]:}y |tjj|t|jd W q tk
rP Y qX qW ||\}}|rxXt||D ]J}t |\}} |j
dr| r|r|d||f 7 }n
j | j | qrW t
jdd |S dS d S )
z#Process the contents of a PyPI pagec s | j jrtttjj| t jd jd}t|dkrd|d krt |d }t
|d }d j j |j
i | < t|t|fS dS )Nr8 r; r< r9 r T)NN)r3 r r r r r# r@ rY rA r r r
setdefaultr1 r )r rC pkgver)r r& r' scan s " z(PackageIndex.process_index..scanr9 z.pyz
#egg=%s-%sc S s d| j ddd S )Nz%sr9 r! r; )rO )mr& r& r' s z,PackageIndex.process_index..r N)r r} r r# r r rO r$ r rJ r2 need_version_infoscan_urlPYPI_MD5sub)
r rB r r rN r r new_urlr5 fragr& )r r' r s$
zPackageIndex.process_indexc C s | j d| d S )NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_all)r rB r& r& r' r s z PackageIndex.need_version_infoc G s: | j | jkr*|r | j|f| | jd | j| j d S )Nz6Scanning index of all packages (this may take a while))r r r r r )r r rt r& r& r' r s zPackageIndex.scan_allc C s~ | j | j|j d | jj|js:| j | j|j d | jj|jsR| j| x&t| jj|jf D ]}| j | qhW d S )Nr8 ) r r unsafe_namer r ro rT not_found_in_indexr )r requirementrB r& r& r'
find_packages s
zPackageIndex.find_packagesc sR | j | j| x,| |j D ] }||kr.|S | jd|| q W tt| j||S )Nz%s does not match %s)prescanr ro r superr obtain)r r installerrQ ) __class__r& r' r s
zPackageIndex.obtainc C sL |j | jd| |j sH|j tj| td|jjtj j
|f dS )z-
checker is a ContentChecker
z Validating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) r r r r r^ unlinkr r r4 rF rZ )r checkerr_ tfpr& r& r'
check_hash s
zPackageIndex.check_hashc C sT xN|D ]F}| j dks4t| s4|jds4tt|r@| j| q| j j| qW dS )z;Add `urls` to the list that will be prescanned for searchesNzfile:)r r r3 r r r append)r urlsrB r& r& r' add_find_links s
zPackageIndex.add_find_linksc C s" | j rtt| j| j d| _ dS )z7Scan urls scheduled for prescanning (e.g. --find-links)N)r r r r )r r& r& r' r s zPackageIndex.prescanc C s< | |j r| jd }}n | jd }}|||j | j d S )Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))ro r r r r )r r methr r& r& r' r % s
z PackageIndex.not_found_in_indexc C s~ t |tsjt|}|rR| j|jd||}t|\}}|jdrN| j|||}|S tj j
|rb|S t |}t | j
||ddS )a Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
r9 z.pyrS N)r r r
_download_urlrO rJ r2 gen_setupr^ rF r r( r fetch_distribution)r r% tmpdirrD foundr5 rI r& r& r' r: / s
zPackageIndex.downloadc s j d| i d}d
fdd }|rHj j| ||}| r`|dk r`|||}|dkrjdk rzj ||}|dkr| rj| ||}|dkrjdrdpd| nj d| |j|jd S dS ) a| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
zSearching for %sNc s |d kr }x|| j D ]t}|jtkrJ rJ|krjd| d|< q|| ko`|jtkp` }|rj|j}||_tj j
|jr|S qW d S )Nz&Skipping development or system egg: %sr9 ) ro rK r r r
r: rS download_locationr^ rF r )reqenvrQ testloc)
develop_okr skippedsourcer r& r' r i s z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of r zBest match: %s)rS )N)r r r r r cloner ) r r r
force_scanr r Z local_indexrQ r r& )r r r r r r' r Q s0
z PackageIndex.fetch_distributionc C s" | j ||||}|dk r |jS dS )a3 Obtain a file suitable for fulfilling `requirement`
DEPRECATED; use the ``fetch_distribution()`` method now instead. For
backward compatibility, this routine is identical but returns the
``location`` of the downloaded distribution instead of a distribution
object.
N)r rS )r r r r r rQ r& r& r' fetch s zPackageIndex.fetchc
C s t j|}|r*dd t||jdd D p,g }t|dkrtjj|}tjj||krtjj ||}ddl
m } |||st j
|| |}ttjj |dd2} | jd|d j|d jtjj|d f W d Q R X |S |rtd ||f ntd
d S ) Nc S s g | ]}|j r|qS r& )rU )rb dr& r& r'
s z*PackageIndex.gen_setup..r9 r )samefilezsetup.pywzIfrom setuptools import setup
setup(name=%r, version=%r, py_modules=[%r])
zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rM rN r rO rY r^ rF rZ dirnamerh Z setuptools.command.easy_installr shutilcopy2r writerT rU splitextr )
r r_ rI r rN r rZ dstr r r& r& r' r s2
zPackageIndex.gen_setupi c C s | j d| d }ztj|}| j|}t|tjjrJtd||j |j
f |j }d}| j }d}d|krt |d} t
tt| }| j||||| t|dZ}
xD|j|} | r|j| |
j| |d7 }| j||||| qP qW | j|||
W d Q R X |S |r|j X d S ) NzDownloading %szCan't download %s: %s %sr r9 zcontent-lengthzContent-Lengthwbr= )r r r r r r r r r r r dl_blocksizer maxr int
reporthookr r r r r r ) r rB r_ fpr r blocknumbssizesizesr r r& r& r' _download_to s:
zPackageIndex._download_toc C s d S )Nr& )r rB r_ r& Zblksizer( r& r& r' r$ s zPackageIndex.reporthookc C s |j drt|S y t|| jS ttjfk
r } z>djdd |jD }|r^| j || nt
d||f W Y d d }~X n t j j
k
r } z|S d }~X n t j jk
r } z,|r| j ||j nt
d||jf W Y d d }~X n tjk
r8 } z.|r| j ||j nt
d||jf W Y d d }~X nP tjtj fk
r } z*|rf| j || nt
d||f W Y d d }~X nX d S )Nzfile: c S s g | ] }t |qS r& )r )rb argr& r& r' r s z)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r3
local_openopen_with_authr r$ r
InvalidURLrh rt r r r r r URLErrorreason
BadStatusLineline
HTTPExceptionsocket)r rB warningvr r& r& r' r s6
" zPackageIndex.open_urlc C s t |\}}|r4x&d|kr0|jddjdd}qW nd}|jdrN|d d }tjj||}|dksn|jd rz| j||S |d
ks|jd r| j||S |jd r| j ||S |d
krt
j j t
j
j|d S | j|d | j||S d S )Nz...\_Z__downloaded__z.egg.zipr- svnzsvn+gitzgit+zhg+r r; Tr0 )rJ replacer2 r^ rF rh r3
_download_svn
_download_git _download_hgr r url2pathnamer# r? r _attempt_download)r rD rB r r4 rI r_ r& r& r' r s$
zPackageIndex._download_urlc C s | j |d d S )NT)r )r rB r& r& r' r < s zPackageIndex.scan_urlc C s6 | j ||}d|jddj kr.| j|||S |S d S )Nr z content-typer )r* r r1 _download_html)r rB r_ r r& r& r' rB ? s z PackageIndex._attempt_downloadc C sl t |}x@|D ]8}|j rtjd|rD|j tj| | j||S P qW |j tj| td| d S )Nz ([^- ]+ - )?Revision \d+:z Unexpected HTML page found at ) r r ra r r r^ r r> r )r rB r r_ r r3 r& r& r' rC F s
zPackageIndex._download_htmlc C s |j ddd }d}|j jdrd|krtjj|\}}}}}} | r|jdrd|d d kr|d d j dd\}}t|\}
} |
rd
|
kr|
j d
d\} }
d | |
f }nd |
}| }|||||| f}tjj|}| jd
|| t j
d|||f |S )Nr< r9 r r zsvn:@z//r8 r; :z --username=%s --password=%sz --username=z'Doing subversion checkout from %s to %szsvn checkout%s -q %s %s) rA r1 r3 r r# r? r
urlunparser r^ system)r rB r_ credsrD netlocrF rc qr authhostuserpwrC r& r& r' r> U s$ zPackageIndex._download_svnc C sp t jj| \}}}}}|jddd }|jddd }d }d|krR|jdd\}}t jj||||df} | |fS )N+r9 r< r rD r r= )r r# urlsplitrA rsplit
urlunsplit)rB
pop_prefixrD rI rF rH r revr& r& r' _vcs_split_rev_from_urlj s z$PackageIndex._vcs_split_rev_from_urlc C sl |j ddd }| j|dd\}}| jd|| tjd||f |d k rh| jd| tjd ||f |S )
Nr< r9 r T)rS z Doing git clone from %s to %szgit clone --quiet %s %szChecking out %sz"(cd %s && git checkout --quiet %s))rA rU r r^ rG )r rB r_ rT r& r& r' r? | s
zPackageIndex._download_gitc C sl |j ddd }| j|dd\}}| jd|| tjd||f |d k rh| jd| tjd ||f |S )
Nr< r9 r T)rS z Doing hg clone from %s to %szhg clone --quiet %s %szUpdating to %sz (cd %s && hg up -C -r %s -q))rA rU r r^ rG )r rB r_ rT r& r& r' r@ s
zPackageIndex._download_hgc G s t j|f| d S )N)r r )r r rt r& r& r' r s zPackageIndex.debugc G s t j|f| d S )N)r r )r r rt r& r& r' r s zPackageIndex.infoc G s t j|f| d S )N)r r )r r rt r& r& r' r s zPackageIndex.warnr )r rV NT)F)F)F)N)N)FFFN)FF)N)F)(r r r r r r r r r r r r r r r r r r r r: r r r r! r* r$ r r r rB rC r> staticmethodrU r? r@ r r r
__classcell__r& r& )r r' r , sL
3
+
#
J
)$
# z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s( t | ts| S | dkr tj| S t| S )N )r r# r unichrchr)cr& r& r' uchr s
r] c C sd | j d}|jdr(t|dd d}n4|jdrDt|dd }ntjjjj|| j d}t|S )Nr9 z#xr; r+ r< r ) rO r3 r# r rk
html_entitiesname2codepointr r] )rN whatr& r& r'
decode_entity s
ra c C s
t t| S )z'Decode HTML entities in the given text.)
entity_subra )textr& r& r' r s r c s fdd}|S )Nc s fdd}|S )Nc s. t j }t j z
| |S t j| X d S )N)r5 getdefaulttimeoutsetdefaulttimeout)rt ru Z old_timeout)rv timeoutr& r' _socket_timeout s
z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr& )rv rg )rf )rv r' rg s z'socket_timeout.<locals>._socket_timeoutr& )rf rg r& )rf r' socket_timeout s rh c C s2 t jj| }|j }tj|}|j }|jddS )aq
A function compatible with Python 2.3-3.3 that will encode
auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
r )r r# r@ encodebase64 encodestringr r= )rK Zauth_sZ
auth_bytesZ
encoded_bytesencodedr& r& r' _encode_auth s
rn c @ s( e Zd ZdZdd Zdd Zdd ZdS )
Credentialz:
A username/password pair. Use like a namedtuple.
c C s || _ || _d S )N)usernamepassword)r rp rq r& r& r' r s zCredential.__init__c c s | j V | jV d S )N)rp rq )r r& r& r' __iter__ s zCredential.__iter__c C s dt | S )Nz%(username)s:%(password)s)vars)r r& r& r' __str__ s zCredential.__str__N)r r r r r rr rt r& r& r& r' ro s ro c @ s0 e Zd Zdd Zedd Zdd Zdd Zd S )
PyPIConfigc C sP t jdddgd}tjj| | tjjtjjdd}tjj |rL| j
| dS )z%
Load from ~/.pypirc
rp rq
repositoryr ~z.pypircN) dictfromkeysr RawConfigParserr r^ rF rh
expanduserr r )r defaultsrcr& r& r' r s
zPyPIConfig.__init__c s& fdd j D }tt j|S )Nc s g | ]} j |d j r|qS )rv )r r )rb section)r r& r' r s z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)sectionsrx r _get_repo_cred)r Zsections_with_repositoriesr& )r r' creds_by_repository s z PyPIConfig.creds_by_repositoryc C s6 | j |dj }|t| j |dj | j |dj fS )Nrv rp rq )r r ro )r r~ repor& r& r' r s zPyPIConfig._get_repo_credc C s* x$| j j D ]\}}|j|r |S q W dS )z
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N)r itemsr3 )r rB rv credr& r& r' find_credential s
zPyPIConfig.find_credentialN)r r r r propertyr r r r& r& r& r' ru s ru c C s6 t jj| \}}}}}}|jdr,tjd|d
krBt|\}} nd}|szt j| }
|
rzt |
}|
j
| f} t j d | |rdt
| }|| ||||f} t jj| }
t jj|
}|jd| n t jj| }|jd t ||}|r2t jj|j\}}}}}}||kr2|| kr2||||||f} t jj| |_|S ) z4Open a urllib2 request, handling HTTP authenticationrE znonnumeric port: ''httphttpsN*Authenticating as %s for %s (from .pypirc)zBasic
Authorizationz
User-Agent)r r )r )r r# r? r2 r r/ r ru r r rp r r rn rF r Request
add_header
user_agentrB )rB r rD rI rF paramsrH r rK rL r r rC r r r% s2h2path2Zparam2Zquery2Zfrag2r& r& r' r. s6
r. c C s | S )Nr& )rB r& r& r'
fix_sf_urlQ s r c C s t jj| \}}}}}}t jj|}tjj|r<t jj| S |j drtjj
|rg }xtj |D ]b} tjj || }
| dkrt
|
d} | j } W dQ R X P ntjj
|
r| d7 } |jdj| d qbW d}
|
j| dj |d } d\}}n
d\}}} ddi}tj| }t jj| ||||S )z7Read a local path, with special support for directoriesr8 z
index.htmlrNz<a href="{name}">{name}</a>)r4 zB<html><head><title>{url}{files} |