[Python-checkins] distutils2: Added PyPIDistribution, started refactoring SimpleIndex with this. WIP
tarek.ziade
python-checkins at python.org
Sun Jul 4 11:48:39 CEST 2010
tarek.ziade pushed fd3496bd8989 to distutils2:
http://hg.python.org/distutils2/rev/fd3496bd8989
changeset: 308:fd3496bd8989
user: Alexis Metaireau <ametaireau at gmail.com>
date: Tue Jun 15 19:39:32 2010 +0200
summary: Added PyPIDistribution, started refactoring SimpleIndex with this. WIP
files: docs/source/index.rst, src/distutils2/pypi/dist.py, src/distutils2/pypi/errors.py, src/distutils2/pypi/oldsimple.py, src/distutils2/pypi/simple.py, src/distutils2/tests/test_pypi_simple.py
diff --git a/docs/source/index.rst b/docs/source/index.rst
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -14,6 +14,8 @@
metadata
new_commands
test_framework
+ pypi
+ version
Indices and tables
==================
diff --git a/src/distutils2/pypi/dist.py b/src/distutils2/pypi/dist.py
new file mode 100644
--- /dev/null
+++ b/src/distutils2/pypi/dist.py
@@ -0,0 +1,100 @@
+"""distutils2.pypi.dist
+
+Provides the PyPIDistribution class thats represents a distribution retreived
+on PyPI.
+"""
+import re
+import urlparse
+
+EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split()
+MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$')
+
+class PyPIDistribution(object):
+ """Represents a distribution retreived from PyPI.
+
+ This is a simple container for various attributes as name, version,
+ location, url etc.
+
+ The PyPIDistribution class is used by the pypi.*Index class to return
+ information about distributions.
+ """
+
+ @classmethod
+ def from_url(cls, url, probable_dist_name=None):
+ """Build a Distribution from a url archive (egg or zip or tgz).
+
+ :param url: complete url of the distribution
+ :param probable_dist_name: the probable name of the distribution. This
+ could be useful when multiple name can be assumed from the archive
+ name.
+ """
+ # if the url contains a md5 hash, get it.
+ md5_hash = None
+ match = MD5_HASH.match(url)
+ if match is not None:
+ md5_hash = match.group(1)
+ # remove the hash
+ url = url.replace("#md5=%s" % md5_hash, "")
+
+ # parse the archive name to find dist name and version
+ archive_name = urlparse.urlparse(url).path.split('/')[-1]
+ extension_matched = False
+ # remove the extension from the name
+ for ext in EXTENSIONS:
+ if archive_name.endswith(ext):
+ archive_name = archive_name[:-len(ext)]
+ extension_matched = True
+
+ name = None
+ # get the name from probable_dist_name
+ if probable_dist_name is not None:
+ if probable_dist_name in archive_name:
+ name = probable_dist_name
+
+ if name is None:
+ version = archive_name.split("-")[-1]
+ name = archive_name[:-len(version)+1]
+
+ if extension_matched is True:
+ return PyPIDistribution(name, version, url=url, md5_hash=md5_hash)
+ raise Exception("test")
+
+ def __init__(self, name, version, type=None, url=None, md5_hash=None):
+ self.name = name
+ self.version = version
+ self.url = url
+ self.type = type
+ self.md5_hash = md5_hash
+ # set the downloaded path to Null by default. The goal here
+ # is to not download distributions multiple times
+ self._downloaded_path = None
+
+ def download(self, url=None, path=None):
+ """Download the distribution to a path, and return it.
+
+ If the path is given in path, use this, otherwise, generates a new one
+ """
+ if url is not None:
+ self.url = url
+
+ # if we do not have downloaded it yet, do it.
+ if self._downloaded_path is None:
+ # download logic goes here
+ real_path = ""
+ return real_path
+
+
+class PyPIDistributions(list):
+ """A container of PyPIDistribution objects.
+
+ Contains methods and facilities to sort and filter distributions.
+ """
+
+ def filter(self, predicate):
+ """Filter the distributions and return just the one matching the given
+ predicate.
+ """
+ dists = self._distributions
+ return filter(predicate.match,
+ [d.version for d in dists if d.name == predicate.name])
+
diff --git a/src/distutils2/pypi/errors.py b/src/distutils2/pypi/errors.py
new file mode 100644
--- /dev/null
+++ b/src/distutils2/pypi/errors.py
@@ -0,0 +1,10 @@
+"""distutils2.pypi.errors
+
+All errors and exceptions raised by PyPiIndex classes.
+"""
+from distutils2.errors import DistutilsError
+
+
+class PyPIError(DistutilsError):
+ """The base class for errors of the pypi python package.
+ """
diff --git a/src/distutils2/pypi/oldsimple.py b/src/distutils2/pypi/oldsimple.py
new file mode 100644
--- /dev/null
+++ b/src/distutils2/pypi/oldsimple.py
@@ -0,0 +1,806 @@
+"""distutils2.pypi.package_index
+
+Implement a simple way to use the PyPI API's, by using known URIs from
+PyPI. Works with the API defined on
+http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api ,
+and available at http://pypi.python.org/simple/ .
+
+"""
+# XXX Make a explaination document on how this API is made.
+
+import sys
+import os.path
+import re
+import urlparse
+import urllib2
+import shutil
+import socket
+import cStringIO
+import httplib
+import logging
+import tempfile
+
+from distutils2.errors import DistutilsError
+from distutils2 import __version__ as __distutils2_version__
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+from fnmatch import translate
+
+# -- Constants ----------------------------------------------------------------
+
+__all__ = ['PackageIndex',
+ 'distros_for_url',
+ 'parse_bdist_wininst',
+ 'interpret_distro_name',
+]
+
+_SOCKET_TIMEOUT = 15
+SOURCE_DIST = 1
+EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
+HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
+PYPI_MD5 = re.compile(
+ '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)'
+ 'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)'
+)
+URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
+EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
+
+# -- Dependencies from distribute ---------------------------------------------
+
+# XXX Remove these dependencies, refactor the package_index codebase to not
+# depends anymore on distribute classes and internal workings.
+from pkg_resources import *
+
+# -- Base functions -----------------------------------------------------------
+
+def parse_bdist_wininst(name):
+ """Parse a bdist_win package name, and return it's package base name
+ and version number if possible.
+
+ Return a tuple of (base,pyversion) for possible .exe names"""
+
+ lower = name.lower()
+ base, py_ver = None, None
+
+ if lower.endswith('.exe'):
+ if lower.endswith('.win32.exe'):
+ base = name[:-10]
+ elif lower.startswith('.win32-py',-16):
+ py_ver = name[-7:-4]
+ base = name[:-16]
+
+ return base,py_ver
+
+def egg_info_for_url(url):
+ """
+ """
+ scheme, server, path, parameters, query, fragment = urlparse.urlparse(url)
+ base = urllib2.unquote(path.split('/')[-1])
+ if '#' in base:
+ base, fragment = base.split('#',1)
+ return base,fragment
+
+def distros_for_url(url, metadata=None):
+ """Yield egg or source distribution objects that might be found at a URL"""
+ base, fragment = egg_info_for_url(url)
+ for dist in distros_for_location(url, base, metadata):
+ yield dist
+ if fragment:
+ match = EGG_FRAGMENT.match(fragment)
+ if match:
+ for dist in interpret_distro_name(url, match.group(1), metadata,
+ precedence = CHECKOUT_DIST):
+ yield dist
+
+def distros_for_location(location, basename, metadata=None):
+ """Yield egg or source distribution objects based on basename"""
+ if basename.endswith('.egg.zip'):
+ basename = basename[:-4] # strip the .zip
+ if basename.endswith('.egg') and '-' in basename:
+ # only one, unambiguous interpretation
+ return [Distribution.from_location(location, basename, metadata)]
+
+ if basename.endswith('.exe'):
+ win_base, py_ver = parse_bdist_wininst(basename)
+ if win_base is not None:
+ return interpret_distro_name(location, win_base, metadata, py_ver,
+ BINARY_DIST, "win32")
+
+ # Try source distro extensions (.zip, .tgz, etc.)
+ for ext in EXTENSIONS:
+ if basename.endswith(ext):
+ basename = basename[:-len(ext)]
+ return interpret_distro_name(location, basename, metadata)
+ return [] # no extension matched
+
+def distros_for_filename(filename, metadata=None):
+ """Yield possible egg or source distribution objects based on a
+ filename.
+ """
+ return distros_for_location(normalize_path(filename),
+ os.path.basename(filename), metadata)
+
+def interpret_distro_name(location, basename, metadata,
+ py_version=None, precedence=SOURCE_DIST, platform=None):
+ """Generate alternative interpretations of a source distro name
+
+ Note: if `location` is a filesystem filename, you should call
+ ``pkg_resources.normalize_path()`` on it before passing it to this
+ routine!
+ """
+ # Generate alternative interpretations of a source distro name
+ # Because some packages are ambiguous as to name/versions split
+ # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
+ # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
+ # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice,
+ # the spurious interpretations should be ignored, because in the event
+ # there's also an "adns" package, the spurious "python-1.1.0" version will
+ # compare lower than any numeric version number, and is therefore unlikely
+ # to match a request for it. It's still a potential problem, though, and
+ # in the long run PyPI and the distutils should go for "safe" names and
+ # versions in distribution archive names (sdist and bdist).
+
+ parts = basename.split('-')
+ if not py_version:
+ for i,p in enumerate(parts[2:]):
+ if len(p)==5 and p.startswith('py2.'):
+ return # It's a bdist_dumb, not an sdist -- bail out
+
+ for p in range(1,len(parts)+1):
+ yield Distribution(
+ location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
+ py_version=py_version, precedence = precedence,
+ platform = platform)
+
+REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
+
+def find_external_links(url, page):
+ """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
+
+ for match in REL.finditer(page):
+ tag, rel = match.groups()
+ rels = map(str.strip, rel.lower().split(','))
+ if 'homepage' in rels or 'download' in rels:
+ for match in HREF.finditer(tag):
+ yield urlparse.urljoin(url, htmldecode(match.group(1)))
+
+ for tag in ("<th>Home Page", "<th>Download URL"):
+ pos = page.find(tag)
+ if pos!=-1:
+ match = HREF.search(page,pos)
+ if match:
+ yield urlparse.urljoin(url, htmldecode(match.group(1)))
+
+user_agent = "Python-urllib/%s distutils2/%s" % (
+ sys.version[:3], __distutils2_version__
+)
+
+class SimpleIndex(Environment):
+ """A distribution index that scans web pages for download URLs"""
+
+ def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',),
+ follow_externals=True, *args, **kw):
+ self._init_logging()
+ Environment.__init__(self,*args,**kw)
+ self.index_url = index_url + "/"[:not index_url.endswith('/')]
+ self.scanned_urls = {}
+ self.fetched_urls = {}
+ self.package_pages = {}
+ self.allows = re.compile('|'.join(map(translate,hosts))).match
+ self.to_scan = []
+ self.follow_externals = follow_externals
+
+ def _init_logging(self):
+ import sys
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+ def process_url(self, url, retrieve=False):
+ """Evaluate a URL as a possible download, and maybe retrieve it"""
+
+ if url in self.scanned_urls and not retrieve:
+ return
+ self.scanned_urls[url] = True
+ if not URL_SCHEME(url):
+ self.process_filename(url)
+ return
+ else:
+ dists = list(distros_for_url(url))
+ if dists:
+ if not self.url_ok(url):
+ return
+ self.debug("Found link: %s", url)
+
+ if dists or not retrieve or url in self.fetched_urls:
+ map(self.add, dists)
+ return # don't need the actual page
+
+ if not self.url_ok(url):
+ self.fetched_urls[url] = True
+ return
+
+ self.info("Reading %s", url)
+ f = self.open_url(url, "Download error: %s -- Some packages may not be found!")
+ if f is None: return
+ self.fetched_urls[url] = self.fetched_urls[f.url] = True
+
+ if 'html' not in f.headers.get('content-type', '').lower():
+ f.close() # not html, we can't process it
+ return
+
+ base = f.url # handle redirects
+ page = f.read()
+ if sys.version_info >= (3,):
+ charset = f.headers.get_param('charset') or 'latin-1'
+ page = page.decode(charset, "ignore")
+ f.close()
+
+ if url.startswith(self.index_url):
+ if self.follow_externals:
+ for match in HREF.finditer(page):
+ link = urlparse.urljoin(base, htmldecode(match.group(1)))
+ self.process_url(link)
+ if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
+ page = self.process_index(url, page)
+
+ def process_filename(self, fn, nested=False):
+ # process filenames or directories
+ if not os.path.exists(fn):
+ self.warn("Not found: %s", fn)
+ return
+
+ if os.path.isdir(fn) and not nested:
+ path = os.path.realpath(fn)
+ for item in os.listdir(path):
+ self.process_filename(os.path.join(path,item), True)
+
+ dists = distros_for_filename(fn)
+ if dists:
+ self.debug("Found: %s", fn)
+ map(self.add, dists)
+
+ def url_ok(self, url, fatal=False):
+ s = URL_SCHEME(url)
+ if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]):
+ return True
+ msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n"
+ if fatal:
+ raise DistutilsError(msg % url)
+ else:
+ self.warn(msg, url)
+ return False
+
+ def scan_egg_links(self, search_path):
+ for item in search_path:
+ if os.path.isdir(item):
+ for entry in os.listdir(item):
+ if entry.endswith('.egg-link'):
+ self.scan_egg_link(item, entry)
+
+ def scan_egg_link(self, path, entry):
+ lines = filter(None, map(str.strip, open(os.path.join(path, entry))))
+ if len(lines)==2:
+ for dist in find_distributions(os.path.join(path, lines[0])):
+ dist.location = os.path.join(path, *lines)
+ dist.precedence = SOURCE_DIST
+ self.add(dist)
+
+ def process_index(self,url,page):
+ """Process the content of PyPI page. For each link, if it's a PyPI one,
+ browse it's page, and satinize the output.
+ """
+ def scan(link):
+ # Process a URL to see if it's for a package page
+ if link.startswith(self.index_url):
+ parts = map(
+ urllib2.unquote, link[len(self.index_url):].split('/')
+ )
+ if len(parts)==2 and '#' not in parts[1]:
+ # it's a package page, sanitize and index it
+ pkg = safe_name(parts[0])
+ ver = safe_version(parts[1])
+ self.package_pages.setdefault(pkg.lower(),{})[link] = True
+ return to_filename(pkg), to_filename(ver)
+ return None, None
+
+ # process an index page into the package-page index
+ for match in HREF.finditer(page):
+ try:
+ scan( urlparse.urljoin(url, htmldecode(match.group(1))) )
+ except ValueError:
+ pass
+
+ pkg, ver = scan(url) # ensure this page is in the page index
+ if pkg:
+ # process individual package page
+ for new_url in find_external_links(url, page):
+ # Process the found URL
+ base, frag = egg_info_for_url(new_url)
+ if base.endswith('.py') and not frag:
+ if ver:
+ new_url+='#egg=%s-%s' % (pkg,ver)
+ else:
+ self.need_version_info(url)
+ self.scan_url(new_url)
+
+ return PYPI_MD5.sub(
+ lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1,3,2), page
+ )
+ else:
+ return "" # no sense double-scanning non-package pages
+
+ def need_version_info(self, url):
+ self.scan_all(
+ "Page at %s links to .py file(s) without version info; an index "
+ "scan is required.", url
+ )
+
+ def scan_all(self, msg=None, *args):
+ if self.index_url not in self.fetched_urls:
+ if msg: self.warn(msg,*args)
+ self.info(
+ "Scanning index of all packages (this may take a while)"
+ )
+ self.scan_url(self.index_url)
+
+ def find_packages(self, requirement):
+ self.scan_url(self.index_url + requirement.unsafe_name+'/')
+
+ if not self.package_pages.get(requirement.key):
+ # Fall back to safe version of the name
+ self.scan_url(self.index_url + requirement.project_name+'/')
+
+ if not self.package_pages.get(requirement.key):
+ # We couldn't find the target package, so search the index page too
+ self.not_found_in_index(requirement)
+
+ for url in list(self.package_pages.get(requirement.key,())):
+ # scan each page that might be related to the desired package
+ self.scan_url(url)
+
+ def obtain(self, requirement, installer=None):
+ self.prescan(); self.find_packages(requirement)
+ for dist in self[requirement.key]:
+ if dist in requirement:
+ return dist
+ self.debug("%s does not match %s", requirement, dist)
+ return super(SimpleIndex, self).obtain(requirement,installer)
+
+ def check_md5(self, cs, info, filename, tfp):
+ if re.match('md5=[0-9a-f]{32}$', info):
+ self.debug("Validating md5 checksum for %s", filename)
+ if cs.hexdigest()<>info[4:]:
+ tfp.close()
+ os.unlink(filename)
+ raise DistutilsError(
+ "MD5 validation failed for "+os.path.basename(filename)+
+ "; possible download problem?")
+
+ def add_find_links(self, urls):
+ """Add `urls` to the list that will be prescanned for searches"""
+ for url in urls:
+ if (
+ self.to_scan is None # if we have already "gone online"
+ or not URL_SCHEME(url) # or it's a local file/directory
+ or url.startswith('file:')
+ or list(distros_for_url(url)) # or a direct package link
+ ):
+ # then go ahead and process it now
+ self.scan_url(url)
+ else:
+ # otherwise, defer retrieval till later
+ self.to_scan.append(url)
+
+ def prescan(self):
+ """Scan urls scheduled for prescanning (e.g. --find-links)"""
+ if self.to_scan:
+ map(self.scan_url, self.to_scan)
+ self.to_scan = None # from now on, go ahead and process immediately
+
+ def not_found_in_index(self, requirement):
+ if self[requirement.key]: # we've seen at least one distro
+ meth, msg = self.info, "Couldn't retrieve index page for %r"
+ else: # no distros seen for this name, might be misspelled
+ meth, msg = (self.warn,
+ "Couldn't find index page for %r (maybe misspelled?)")
+ meth(msg, requirement.unsafe_name)
+ self.scan_all()
+
+ def download(self, spec, tmpdir=None):
+ """Locate and/or download `spec` to `tmpdir`, returning a local path
+
+ `spec` may be a ``Requirement`` object, or a string containing a URL,
+ an existing local filename, or a project/version requirement spec
+ (i.e. the string form of a ``Requirement`` object). If it is the URL
+ of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
+ that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
+ automatically created alongside the downloaded file.
+
+ If `spec` is a ``Requirement`` object or a string containing a
+ project/version requirement spec, this method returns the location of
+ a matching distribution (possibly after downloading it to `tmpdir`).
+ If `spec` is a locally existing file or directory name, it is simply
+ returned unchanged. If `spec` is a URL, it is downloaded to a subpath
+ of `tmpdir`, and the local filename is returned. Various errors may be
+ raised if a problem occurs during downloading.
+ """
+ if tmpdir is None:
+ tmpdir=tempfile.mkdtemp()
+ if not isinstance(spec,Requirement):
+ scheme = URL_SCHEME(spec)
+ if scheme:
+ # It's a url, download it to tmpdir
+ found = self._download_url(scheme.group(1), spec, tmpdir)
+ base, fragment = egg_info_for_url(spec)
+ if base.endswith('.py'):
+ found = self.gen_setup(found,fragment,tmpdir)
+ return found
+ elif os.path.exists(spec):
+ # Existing file or directory, just return it
+ return spec
+ else:
+ try:
+ spec = Requirement.parse(spec)
+ except ValueError:
+ raise DistutilsError(
+ "Not a URL, existing file, or requirement spec: %r" %
+ (spec,)
+ )
+ return getattr(self.fetch_distribution(spec, tmpdir),'location',None)
+
+ def fetch_distribution(self,
+ requirement, tmpdir, force_scan=False, source=False, develop_ok=False
+ ):
+ """Obtain a distribution suitable for fulfilling `requirement`
+
+ `requirement` must be a ``pkg_resources.Requirement`` instance.
+ If necessary, or if the `force_scan` flag is set, the requirement is
+ searched for in the (online) package index as well as the locally
+ installed packages. If a distribution matching `requirement` is found,
+ the returned distribution's ``location`` is the value you would have
+ gotten from calling the ``download()`` method with the matching
+ distribution's URL or filename. If no matching distribution is found,
+ ``None`` is returned.
+
+ If the `source` flag is set, only source distributions and source
+ checkout links will be considered. Unless the `develop_ok` flag is
+ set, development and system eggs (i.e., those using the ``.egg-info``
+ format) will be ignored.
+ """
+
+ # process a Requirement
+ self.info("Searching for %s", requirement)
+ skipped = {}
+
+ def find(req):
+ # Find a matching distribution; may be called more than once
+
+ for dist in self[req.key]:
+
+ if dist.precedence==DEVELOP_DIST and not develop_ok:
+ if dist not in skipped:
+ self.warn("Skipping development or system egg: %s",dist)
+ skipped[dist] = 1
+ continue
+
+ if dist in req and (dist.precedence<=SOURCE_DIST or not source):
+ self.info("Best match: %s", dist)
+ return dist.clone(
+ location=self.download(dist.location, tmpdir)
+ )
+
+ if force_scan:
+ self.prescan()
+ self.find_packages(requirement)
+
+ dist = find(requirement)
+ if dist is None and self.to_scan is not None:
+ self.prescan()
+ dist = find(requirement)
+
+ if dist is None and not force_scan:
+ self.find_packages(requirement)
+ dist = find(requirement)
+
+ if dist is None:
+ self.warn(
+ "No local packages or download links found for %s%s",
+ (source and "a source distribution of " or ""),
+ requirement,
+ )
+ return dist
+
+ def fetch(self, requirement, tmpdir, force_scan=False, source=False):
+ """Obtain a file suitable for fulfilling `requirement`
+
+ DEPRECATED; use the ``fetch_distribution()`` method now instead. For
+ backward compatibility, this routine is identical but returns the
+ ``location`` of the downloaded distribution instead of a distribution
+ object.
+ """
+ dist = self.fetch_distribution(requirement,tmpdir,force_scan,source)
+ if dist is not None:
+ return dist.location
+ return None
+
+ def gen_setup(self, filename, fragment, tmpdir):
+ match = EGG_FRAGMENT.match(fragment)
+ dists = match and [d for d in
+ interpret_distro_name(filename, match.group(1), None) if d.version
+ ] or []
+
+ if len(dists)==1: # unambiguous ``#egg`` fragment
+ basename = os.path.basename(filename)
+
+ # Make sure the file has been downloaded to the temp dir.
+ if os.path.dirname(filename) != tmpdir:
+ dst = os.path.join(tmpdir, basename)
+ from setuptools.command.easy_install import samefile
+ if not samefile(filename, dst):
+ shutil.copy2(filename, dst)
+ filename=dst
+
+ file = open(os.path.join(tmpdir, 'setup.py'), 'w')
+ file.write(
+ "from setuptools import setup\n"
+ "setup(name=%r, version=%r, py_modules=[%r])\n"
+ % (
+ dists[0].project_name, dists[0].version,
+ os.path.splitext(basename)[0]
+ )
+ )
+ file.close()
+ return filename
+
+ elif match:
+ raise DistutilsError(
+ "Can't unambiguously interpret project/version identifier %r; "
+ "any dashes in the name or version should be escaped using "
+ "underscores. %r" % (fragment,dists)
+ )
+ else:
+ raise DistutilsError(
+ "Can't process plain .py without an '#egg=name-version'"
+ " suffix to enable automatic setup script generation."
+ )
+
+ dl_blocksize = 8192
+ def _download_to(self, url, filename):
+ self.info("Downloading %s", url)
+ # Download the file
+ fp, tfp, info = None, None, None
+ try:
+ if '#' in url:
+ url, info = url.split('#', 1)
+ fp = self.open_url(url)
+ if isinstance(fp, urllib2.HTTPError):
+ raise DistutilsError(
+ "Can't download %s: %s %s" % (url, fp.code,fp.msg)
+ )
+ cs = md5()
+ headers = fp.info()
+ blocknum = 0
+ bs = self.dl_blocksize
+ size = -1
+ if "content-length" in headers:
+ size = int(headers["Content-Length"])
+ self.reporthook(url, filename, blocknum, bs, size)
+ tfp = open(filename,'wb')
+ while True:
+ block = fp.read(bs)
+ if block:
+ cs.update(block)
+ tfp.write(block)
+ blocknum += 1
+ self.reporthook(url, filename, blocknum, bs, size)
+ else:
+ break
+ if info: self.check_md5(cs, info, filename, tfp)
+ return headers
+ finally:
+ if fp: fp.close()
+ if tfp: tfp.close()
+
+ def reporthook(self, url, filename, blocknum, blksize, size):
+ pass # no-op
+
+ def open_url(self, url, warning=None):
+ if url.startswith('file:'):
+ return local_open(url)
+ try:
+ return open_with_auth(url)
+ except (ValueError, httplib.InvalidURL), v:
+ msg = ' '.join([str(arg) for arg in v.args])
+ if warning:
+ self.warn(warning, msg)
+ else:
+ raise DistutilsError('%s %s' % (url, msg))
+ except urllib2.HTTPError, v:
+ return v
+ except urllib2.URLError, v:
+ if warning:
+ self.warn(warning, v.reason)
+ else:
+ raise DistutilsError("Download error for %s: %s"
+ % (url, v.reason))
+ except httplib.BadStatusLine, v:
+ if warning:
+ self.warn(warning, v.line)
+ else:
+ raise DistutilsError('%s returned a bad status line. '
+ 'The server might be down, %s' % \
+ (url, v.line))
+ except httplib.HTTPException, v:
+ if warning:
+ self.warn(warning, v)
+ else:
+ raise DistutilsError("Download error for %s: %s"
+ % (url, v))
+
+ def _download_url(self, scheme, url, tmpdir):
+ # Determine download filename
+ name = filter(None,urlparse.urlparse(url)[2].split('/'))
+ if name:
+ name = name[-1]
+ while '..' in name:
+ name = name.replace('..','.').replace('\\','_')
+ else:
+ name = "__downloaded__" # default if URL has no path contents
+
+ if name.endswith('.egg.zip'):
+ name = name[:-4] # strip the extra .zip before download
+
+ filename = os.path.join(tmpdir,name)
+
+ # Download the file
+ if scheme=='svn' or scheme.startswith('svn+'):
+ return self._download_svn(url, filename)
+ elif scheme=='file':
+ return urllib2.url2pathname(urlparse.urlparse(url)[2])
+ else:
+ self.url_ok(url, True) # raises error if not allowed
+ return self._attempt_download(url, filename)
+
+ def scan_url(self, url):
+ self.process_url(url, True)
+
+ def _attempt_download(self, url, filename):
+ headers = self._download_to(url, filename)
+ if 'html' in headers.get('content-type','').lower():
+ return self._download_html(url, headers, filename)
+ else:
+ return filename
+
+ def _download_html(self, url, headers, filename):
+ file = open(filename)
+ for line in file:
+ if line.strip():
+ # Check for a subversion index page
+ if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
+ # it's a subversion index page:
+ file.close()
+ os.unlink(filename)
+ return self._download_svn(url, filename)
+ break # not an index page
+ file.close()
+ os.unlink(filename)
+ raise DistutilsError("Unexpected HTML page found at "+url)
+
+ def _download_svn(self, url, filename):
+ url = url.split('#',1)[0] # remove any fragment for svn's sake
+ self.info("Doing subversion checkout from %s to %s", url, filename)
+ os.system("svn checkout -q %s %s" % (url, filename))
+ return filename
+
+ def debug(self, msg, *args):
+ logging.debug(msg, *args)
+
+ def info(self, msg, *args):
+ logging.info(msg, *args)
+
+ def warn(self, msg, *args):
+ logging.warn(msg, *args)
+
+# This pattern matches a character entity reference (a decimal numeric
+# references, a hexadecimal numeric reference, or a named reference).
+entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
+
+def uchr(c):
+ if not isinstance(c, int):
+ return c
+ if c>255:
+ return unichr(c)
+ return chr(c)
+
+def decode_entity(match):
+ what = match.group(1)
+ if what.startswith('#x'):
+ what = int(what[2:], 16)
+ elif what.startswith('#'):
+ what = int(what[1:])
+ else:
+ from htmlentitydefs import name2codepoint
+ what = name2codepoint.get(what, match.group(0))
+ return uchr(what)
+
+def htmldecode(text):
+ """Decode HTML entities in the given text."""
+ return entity_sub(decode_entity, text)
+
+def socket_timeout(timeout=15):
+ def _socket_timeout(func):
+ def _socket_timeout(*args, **kwargs):
+ old_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(timeout)
+ try:
+ return func(*args, **kwargs)
+ finally:
+ socket.setdefaulttimeout(old_timeout)
+ return _socket_timeout
+ return _socket_timeout
+
+def _open_url(self, url):
+ """Open a urllib2 request, handling HTTP authentication"""
+
+ scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
+
+ if scheme in ('http', 'https'):
+ auth, host = urllib2.splituser(netloc)
+ else:
+ auth = None
+
+ if auth:
+ auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
+ new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
+ request = urllib2.Request(new_url)
+ request.add_header("Authorization", auth)
+ else:
+ request = urllib2.Request(url)
+
+ request.add_header('User-Agent', user_agent)
+ fp = urllib2.urlopen(request)
+
+ if auth:
+ # Put authentication info back into request URL if same host,
+ # so that links found on the page will work
+ s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
+ if s2==scheme and h2==host:
+ fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))
+
+ return fp
+
+
+# adding a timeout to avoid freezing package_index
+open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
+
+def fix_sf_url(url):
+ return url # backward compatibility
+
+def local_open(url):
+ """Read a local path, with special support for directories"""
+ scheme, server, path, param, query, frag = urlparse.urlparse(url)
+ filename = urllib2.url2pathname(path)
+ if os.path.isfile(filename):
+ return urllib2.urlopen(url)
+ elif path.endswith('/') and os.path.isdir(filename):
+ files = []
+ for f in os.listdir(filename):
+ if f=='index.html':
+ fp = open(os.path.join(filename,f),'rb')
+ body = fp.read()
+ fp.close()
+ break
+ elif os.path.isdir(os.path.join(filename,f)):
+ f+='/'
+ files.append("<a href=%r>%s</a>" % (f,f))
+ else:
+ body = ("<html><head><title>%s</title>" % url) + \
+ "</head><body>%s</body></html>" % '\n'.join(files)
+ status, message = 200, "OK"
+ else:
+ status, message, body = 404, "Path not found", "Not found"
+
+ return urllib2.HTTPError(url, status, message,
+ {'content-type':'text/html'}, cStringIO.StringIO(body))
diff --git a/src/distutils2/pypi/simple.py b/src/distutils2/pypi/simple.py
--- a/src/distutils2/pypi/simple.py
+++ b/src/distutils2/pypi/simple.py
@@ -1,34 +1,99 @@
"""pypi.simple
-Contains the class "SimpleIndex" to request the PyPI "simple" index.
+Contains the class "SimpleIndex", a simple spider to find and retreive
+distributions on the Python Package Index, using it's "simple" API,
+avalaible at http://pypi.python.org/simple/
"""
+from fnmatch import translate
+import urlparse
+import sys
+import re
+import urllib2
+import socket
try:
from hashlib import md5
except ImportError:
from md5 import md5
from distutils2.version import VersionPredicate
+from distutils2.pypi.dist import PyPIDistribution, EXTENSIONS
+from distutils2.pypi.errors import PyPIError
+from distutils2 import __version__ as __distutils2_version__
-from distutils2.pypi.dist import PyPIDistribution
-from distutils2.pypi.errors import PyPIError
+# -- Constants -----------------------------------------------
+PYPI_DEFAULT_INDEX_URL = "http://pypi.python.org/simple/"
+SOCKET_TIMEOUT=15
+USER_AGENT = "Python-urllib/%s distutils2/%s" % (
+ sys.version[:3], __distutils2_version__
+)
+# -- Regexps -------------------------------------------------
+EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
+HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
+PYPI_MD5 = re.compile(
+ '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)'
+ 'href="[^?]+\?:action=show_md5&digest=([0-9a-f]{32})">md5</a>\\)'
+)
+URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
+# This pattern matches a character entity reference (a decimal numeric
+# references, a hexadecimal numeric reference, or a named reference).
+ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
-PYPI_DEFAULT_INDEX_URL = "http://pypi.python.org/simple/"
+def socket_timeout(timeout=SOCKET_TIMEOUT):
+ """Decorator to add a socket timeout when requesting pages on PyPI.
+ """
+ def _socket_timeout(func):
+ def _socket_timeout(*args, **kwargs):
+ old_timeout = socket.getdefaulttimeout()
+ socket.setdefaulttimeout(timeout)
+ try:
+ return func(*args, **kwargs)
+ finally:
+ socket.setdefaulttimeout(old_timeout)
+ return _socket_timeout
+ return _socket_timeout
+
+
+class UrlProcessor(object):
+ """Tools to process urls and return links from them.
+ """
+ def __init__(self):
+ self._processed_urls = {}
+
+ def find_links(self, url, is_browsable=lambda x:True):
+ """Find links and return them, for a specific url.
+
+ Use "is_browsable" to provides a method to tell if the found urls might
+ be browsed or not.
+ """
+ pass
+
class SimpleIndex(object):
- """
+ """Provides useful tools to request the Python Package Index simple API
"""
def __init__(self, url=PYPI_DEFAULT_INDEX_URL,
- hosts=('*',), ):
+ hosts=('*',), follow_externals=False):
"""Class constructor.
:param index_url: the url of the simple index to search on.
:param hosts: a list of hosts allowed to be considered as internals (to
differenciate with external hosts)
+ :param follow_externals: tell if follow external links is needed or not
+ Default is False
"""
self.index_url = url
- self.allowed_hosts = hosts
+ self.follow_externals=follow_externals
+
+ # create a regexp to match all given hosts
+ self._allowed_hosts = re.compile('|'.join(map(translate,hosts))).match
+
+ # _current_requirements are used to know what the index is currently
+ # browsing for. This could be used to determine the distributon name
+ # while having multiple ambigous names/version couples
+ self._current_requirements = None
+ self._processed_pages = []
def get_distributions(self, requirements):
"""Browse the PyPI to find distributions that fullfil the given
@@ -38,30 +103,57 @@
version specifiers, as described in PEP345.
"""
requirements = VersionPredicate(requirements)
- self._process_package_index_page(requirements.name)
+ self.current_requirements = requirements
+
+ # process the index for this project
+ distributions = self._process_pypi_page(requirements.name)
+
+ # filter with requirements and return the results
+ return distributions.filter(requirements)
- def download(self, requirements, tmp_dir=None):
+ def download(self, requirements, temp_path=None):
"""Download the distribution, using the requirements.
If more than one distribution match the requirements, use the last
version.
- Download the distribution, and put it in the tmp_dir. If no tmp_dir is
- given, creates and return one.
+ Download the distribution, and put it in the temp_path. If no temp_path
+ is given, creates and return one.
Returns the complete absolute path to the downloaded archive.
"""
distributions = self.get_distributions(requirements)
- distribution = self._find_last_distribution(distributions)
-
- # Download logic goes here
+ return distributions.get_last(requirements) \
+ .download(temp_path=temp_path)
- def _browsable(self, url):
+ def _is_browsable(self, url):
"""Tell if the given URL needs to be browsed or not, according to the
- object internal attributes"""
- return True
+ object internal attributes.
- def _process_package_index_page(self, name):
- """Find and process a PyPI page for the given name.
+ It uses the follow_externals and the hosts list to tell if the given
+ url is browsable or not.
+ """
+ if self.follow_externals is True:
+ return True
+ return True if self._allowed_hosts(urlparse(url).netloc) else False
+
+ def _is_distribution(self, link):
+ """Tell if the given URL matches to a distribution name or not.
+ """
+
+
+ def _process_url(self, url):
+ """Process an url and return fetched links.
+ """
+ f = self._open_url(url)
+ base = f.url
+ for match in HREF.finditer(f.read()):
+ link = urlparse.urljoin(base, self._htmldecode(match.group(1)))
+ if self._is_distribution(link):
+ PyPIDistribution.from_url(link)
+ else:
+
+ def _process_pypi_page(self, name):
+ """Find and process a PyPI page for the given project name.
:param name: the name of the project to find the page
"""
@@ -69,9 +161,55 @@
# Put all informations about the processed pages in the
# _processed_pages attribute.
url = self.index_url + name + "/"
+ found_links = self._process_url(url)
# Search for external links here, and process them if needed.
- # use the _browsable method to know if we can process the found links.
for link in found_links:
- if self._browsable(link):
+ if self._is_browsable(link):
self._search_in_url(link)
+
+ @socket_timeout()
+ def _open_url(self, url):
+ """Open a urllib2 request, handling HTTP authentication
+ """
+ scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
+
+ if scheme in ('http', 'https'):
+ auth, host = urllib2.splituser(netloc)
+ else:
+ auth = None
+
+ if auth:
+ auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
+ new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
+ request = urllib2.Request(new_url)
+ request.add_header("Authorization", auth)
+ else:
+ request = urllib2.Request(url)
+
+ request.add_header('User-Agent', USER_AGENT)
+ fp = urllib2.urlopen(request)
+
+ if auth:
+ # Put authentication info back into request URL if same host,
+ # so that links found on the page will work
+ s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
+ if s2==scheme and h2==host:
+ fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))
+
+ return fp
+
+ def _decode_entity(self, match):
+ what = match.group(1)
+ if what.startswith('#x'):
+ what = int(what[2:], 16)
+ elif what.startswith('#'):
+ what = int(what[1:])
+ else:
+ from htmlentitydefs import name2codepoint
+ what = name2codepoint.get(what, match.group(0))
+ return unichr(what)
+
+ def _htmldecode(self, text):
+ """Decode HTML entities in the given text."""
+ return ENTITY_SUB(self._decode_entity, text)
diff --git a/src/distutils2/tests/test_pypi_simple.py b/src/distutils2/tests/test_pypi_simple.py
--- a/src/distutils2/tests/test_pypi_simple.py
+++ b/src/distutils2/tests/test_pypi_simple.py
@@ -79,11 +79,28 @@
'http://www.famfamfam.com/">')
index.process_index(url, page)
- def test_url_ok(self):
- index = simple.SimpleIndex(
- hosts=('www.example.com',))
- url = 'file:///tmp/test_simple'
- self.assertTrue(index.url_ok(url, True))
+ def test_browsable_urls(self):
+ """Test we can given PyPI index a list of allowed hosts
+
+ """
+ index = simple.SimpleIndex(hosts=["pypi.python.org","test.org"])
+ good_urls = (
+ "http://pypi.python.org/foo/bar",
+ "http://pypi.python.org/simple/foobar",
+ "http://test.org",
+ "http://test.org/",
+ "http://test.org/simple/",
+ )
+ bad_urls = (
+ "http://python.org",
+ "http://test.tld",
+ )
+
+ for url in good_urls:
+ self.assertTrue(index._is_browsable(url))
+
+ for url in bad_urls:
+ self.assertFalse(index._is_browsable(url))
@use_pypi_server("test_found_links")
def test_found_links(self, server):
@@ -162,14 +179,6 @@
# md5 hash
self.assertRaises(DistutilsError, pi.download,
simple.Requirement.parse("badmd5"))
- # XXX: is this making sense ? do file on pypi can have a wrong md5
- # hash ?
- # If the pypi package index contain a file with the wrong md5, try to
- # use the external locations to retreive it.
- # in the test, the "wrongmd5" project contain a bad version of the
- # archive, but a external site contains the good one. We want to use the
- # external one to retreive the file with the good md5.
- # if we really can't manage to get the right md5, raise an exception
for path in paths:
shutil.rmtree(os.path.dirname(path))
--
Repository URL: http://hg.python.org/distutils2
More information about the Python-checkins
mailing list