[Python-checkins] distutils2: some more tests + first fonctionnal version
tarek.ziade
python-checkins at python.org
Sun Jul 4 11:48:39 CEST 2010
tarek.ziade pushed bd3fbdf672db to distutils2:
http://hg.python.org/distutils2/rev/bd3fbdf672db
changeset: 309:bd3fbdf672db
user: Alexis Metaireau <ametaireau at gmail.com>
date: Fri Jun 18 02:07:55 2010 +0200
summary: some more tests + first fonctionnal version
files: src/distutils2/pypi/dist.py, src/distutils2/pypi/errors.py, src/distutils2/pypi/simple.py, src/distutils2/tests/test_pypi_dist.py, src/distutils2/version.py
diff --git a/src/distutils2/pypi/dist.py b/src/distutils2/pypi/dist.py
--- a/src/distutils2/pypi/dist.py
+++ b/src/distutils2/pypi/dist.py
@@ -6,6 +6,8 @@
import re
import urlparse
+from distutils2.version import suggest_normalized_version
+
EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split()
MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$')
@@ -49,15 +51,22 @@
# get the name from probable_dist_name
if probable_dist_name is not None:
if probable_dist_name in archive_name:
- name = probable_dist_name
+ name = probable_dist_name
if name is None:
- version = archive_name.split("-")[-1]
- name = archive_name[:-len(version)+1]
+ # determine the name and the version
+ splits = archive_name.split("-")
+ version = splits[-1]
+ name = "-".join(splits[:-1])
+ else:
+ # just determine the version
+ version = archive_name[len(name):]
+ if version.startswith("-"):
+ version = version[1:]
+ version = suggest_normalized_version(version)
if extension_matched is True:
return PyPIDistribution(name, version, url=url, md5_hash=md5_hash)
- raise Exception("test")
def __init__(self, name, version, type=None, url=None, md5_hash=None):
self.name = name
@@ -83,6 +92,10 @@
real_path = ""
return real_path
+ def __str__(self):
+ """string representation of the PyPIDistribution"""
+ return "%s-%s" % (self.name, self.version)
+
class PyPIDistributions(list):
"""A container of PyPIDistribution objects.
@@ -94,7 +107,6 @@
"""Filter the distributions and return just the one matching the given
predicate.
"""
- dists = self._distributions
- return filter(predicate.match,
- [d.version for d in dists if d.name == predicate.name])
+ return [dist for dist in self if dist.name == predicate.name and
+ predicate.match(dist.version)]
diff --git a/src/distutils2/pypi/errors.py b/src/distutils2/pypi/errors.py
--- a/src/distutils2/pypi/errors.py
+++ b/src/distutils2/pypi/errors.py
@@ -6,5 +6,7 @@
class PyPIError(DistutilsError):
- """The base class for errors of the pypi python package.
- """
+ """The base class for errors of the pypi python package."""
+
+class DistributionNotFound(PyPIError):
+ """Raised when no distribution match the given requirements."""
diff --git a/src/distutils2/pypi/simple.py b/src/distutils2/pypi/simple.py
--- a/src/distutils2/pypi/simple.py
+++ b/src/distutils2/pypi/simple.py
@@ -16,8 +16,9 @@
from md5 import md5
from distutils2.version import VersionPredicate
-from distutils2.pypi.dist import PyPIDistribution, EXTENSIONS
-from distutils2.pypi.errors import PyPIError
+from distutils2.pypi.dist import PyPIDistribution, PyPIDistributions, \
+ EXTENSIONS
+from distutils2.pypi.errors import PyPIError, DistributionNotFound
from distutils2 import __version__ as __distutils2_version__
# -- Constants -----------------------------------------------
@@ -54,21 +55,6 @@
return _socket_timeout
-class UrlProcessor(object):
- """Tools to process urls and return links from them.
- """
- def __init__(self):
- self._processed_urls = {}
-
- def find_links(self, url, is_browsable=lambda x:True):
- """Find links and return them, for a specific url.
-
- Use "is_browsable" to provides a method to tell if the found urls might
- be browsed or not.
- """
- pass
-
-
class SimpleIndex(object):
"""Provides useful tools to request the Python Package Index simple API
"""
@@ -89,11 +75,11 @@
# create a regexp to match all given hosts
self._allowed_hosts = re.compile('|'.join(map(translate,hosts))).match
- # _current_requirements are used to know what the index is currently
- # browsing for. This could be used to determine the distributon name
- # while having multiple ambigous names/version couples
- self._current_requirements = None
- self._processed_pages = []
+ # we keep an index of pages we have processed, in order to avoid
+ # scanning them multple time (eg. if there is multiple pages pointing
+ # on one)
+ self._processed_urls = []
+ self._distributions = {}
def get_distributions(self, requirements):
"""Browse the PyPI to find distributions that fullfil the given
@@ -103,13 +89,20 @@
version specifiers, as described in PEP345.
"""
requirements = VersionPredicate(requirements)
- self.current_requirements = requirements
# process the index for this project
- distributions = self._process_pypi_page(requirements.name)
+ self._process_pypi_page(requirements.name)
# filter with requirements and return the results
- return distributions.filter(requirements)
+ if self._distributions.has_key(requirements.name):
+ dists = self._distributions[requirements.name].filter(requirements)
+ else:
+ dists = []
+
+ if dists is not []:
+ return dists
+ else:
+ raise DistributionNotFound(requirements.name)
def download(self, requirements, temp_path=None):
"""Download the distribution, using the requirements.
@@ -134,23 +127,54 @@
"""
if self.follow_externals is True:
return True
- return True if self._allowed_hosts(urlparse(url).netloc) else False
+ return True if self._allowed_hosts(
+ urlparse.urlparse(url).netloc) else False
def _is_distribution(self, link):
"""Tell if the given URL matches to a distribution name or not.
"""
+ for ext in EXTENSIONS:
+ if ext in link:
+ return True
+ return False
+ def _register_dist(self, dist):
+ """Register a distribution as a part of fetched distributions for
+ SimpleIndex.
- def _process_url(self, url):
- """Process an url and return fetched links.
+ Return the PyPIDistributions object for the specified project name
+ """
+ # Internally, check if a entry exists with the project name, if not,
+ # create a new one, and if exists, add the dist to the pool.
+ if not self._distributions.has_key(dist.name):
+ self._distributions[dist.name] = PyPIDistributions()
+ self._distributions[dist.name].append(dist)
+ return self._distributions[dist.name]
+
+ def _process_url(self, url, project_name=None, follow_links=True):
+ """Process an url and search for distributions packages.
+
+ :param url: the url to analyse
+ :param project_name: the project name we are searching for.
+ :param follow_links: We do not want to follow links more than from one
+ level. This parameter tells if we want to follow the links we find (eg.
+ run recursively this method on it)
"""
f = self._open_url(url)
base = f.url
+ self._processed_urls.append(url)
for match in HREF.finditer(f.read()):
link = urlparse.urljoin(base, self._htmldecode(match.group(1)))
- if self._is_distribution(link):
- PyPIDistribution.from_url(link)
- else:
+ if link not in self._processed_urls:
+ if self._is_distribution(link):
+ # it's a distribution, so create a dist object
+ self._processed_urls.append(link)
+ self._register_dist(PyPIDistribution.from_url(link,
+ project_name))
+ else:
+ if self._is_browsable(link) and follow_links:
+ self._process_url(link, project_name,
+ follow_links=False)
def _process_pypi_page(self, name):
"""Find and process a PyPI page for the given project name.
@@ -158,15 +182,8 @@
:param name: the name of the project to find the page
"""
# Browse and index the content of the given PyPI page.
- # Put all informations about the processed pages in the
- # _processed_pages attribute.
url = self.index_url + name + "/"
- found_links = self._process_url(url)
-
- # Search for external links here, and process them if needed.
- for link in found_links:
- if self._is_browsable(link):
- self._search_in_url(link)
+ self._process_url(url, name)
@socket_timeout()
def _open_url(self, url):
@@ -186,7 +203,6 @@
request.add_header("Authorization", auth)
else:
request = urllib2.Request(url)
-
request.add_header('User-Agent', USER_AGENT)
fp = urllib2.urlopen(request)
diff --git a/src/distutils2/tests/test_pypi_dist.py b/src/distutils2/tests/test_pypi_dist.py
new file mode 100644
--- /dev/null
+++ b/src/distutils2/tests/test_pypi_dist.py
@@ -0,0 +1,68 @@
+"""Tests for the distutils2.pypi.dist module.
+"""
+
+from distutils2.pypi.dist import PyPIDistribution, PyPIDistributions
+from distutils2.tests.support import unittest
+from distutils2.version import VersionPredicate
+
+class TestPyPIDistribution(unittest.TestCase):
+ """tests the pypi.dist.PyPIDistribution class"""
+
+ def test_instanciation(self):
+ """Test the Distribution class provides us the good attributes when
+ given on construction"""
+ dist = PyPIDistribution("FooBar", "1.1")
+ self.assertEqual("FooBar", dist.name)
+ self.assertEqual("1.1", dist.version)
+
+ def test_from_url(self):
+ """Test that the Distribution object can be built from a single URL"""
+ url_list = {
+ 'FooBar-1.1.0.tar.gz': {
+ 'name': 'FooBar',
+ 'version': '1.1.0',
+ },
+ 'Foo-Bar-1.1.0.zip': {
+ 'name': 'Foo-Bar',
+ 'version': '1.1.0',
+ },
+ 'foobar-1.1b2.tar.gz#md5=123123123123123': {
+ 'name': 'foobar',
+ 'version': '1.1b2',
+ 'url':'http://test.tld/foobar-1.1b2.tar.gz', #without md5 hash
+ 'md5_hash': '123123123123123',
+ }
+ }
+
+ for url, attributes in url_list.items():
+ dist = PyPIDistribution.from_url("http://test.tld/"+url)
+ for attribute, value in attributes.items():
+ self.assertEqual(getattr(dist, attribute), value)
+
+class TestPyPIDistributions(unittest.TestCase):
+ """test the pypi.distr.PyPIDistributions class"""
+
+ def test_filter(self):
+ """Test we filter the distributions the right way, using version
+ predicate match method"""
+ dists = PyPIDistributions((
+ PyPIDistribution("FooBar", "1.1"),
+ PyPIDistribution("FooBar", "1.1.1"),
+ PyPIDistribution("FooBar", "1.2"),
+ PyPIDistribution("FooBar", "1.2.1"),
+ ))
+ filtered = dists.filter(VersionPredicate("FooBar (<1.2)"))
+ self.assertNotIn(dists[2], filtered)
+ self.assertNotIn(dists[3], filtered)
+ self.assertIn(dists[0], filtered)
+ self.assertIn(dists[1], filtered)
+
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestPyPIDistribution))
+ suite.addTest(unittest.makeSuite(TestPyPIDistributions))
+ return suite
+
+if __name__ == '__main__':
+ unittest.main(defaultTest="test_suite")
diff --git a/src/distutils2/version.py b/src/distutils2/version.py
--- a/src/distutils2/version.py
+++ b/src/distutils2/version.py
@@ -215,7 +215,7 @@
on observation of versions currently in use on PyPI. Given a dump of
those version during PyCon 2009, 4287 of them:
- 2312 (53.93%) match NormalizedVersion without change
- - with the automatic suggestion
+ with the automatic suggestion
- 3474 (81.04%) match when using this suggestion method
@param s {str} An irrational version string.
--
Repository URL: http://hg.python.org/distutils2
More information about the Python-checkins
mailing list