Commits

Vinay Sajip committed 070f49c

Improved access to requirements data, refined tests w.r.t. EggInfoDistribution.

Comments (0)

Files changed (7)

distlib/database.py

 from .compat import StringIO, configparser
 from .version import get_scheme, UnsupportedVersionError
 from .metadata import Metadata
-from .util import (parse_requires, cached_property, get_export_entry,
-                   get_extended_metadata)
+from .util import parse_requires, cached_property, get_export_entry
 
 
 __all__ = ['Distribution', 'BaseInstalledDistribution',
     def download_url(self):
         return self.metadata.download_url
 
-    @cached_property
-    def extended_metadata(self):
-        result = get_extended_metadata(self.name, self.version)
-        # put relevant things in base metadata.
-        reqts = result.get('requirements', {})
-        deps = []
-        deps.extend(reqts.get('install', []))
-        if self.metadata['Requires']:
-            pass # import pdb; pdb.set_trace()
-        else:
-            self.metadata['Requires'] = deps
-        return result
-
     def get_requirements(self, key):
+        """
+        Get the requirements of a particular type
+        ('setup', 'install', 'test')
+        """
         result = []
-        emd = self.extended_metadata
-        if 'requirements' in emd and key in emd['requirements']:
-            result = emd['requirements'][key]
+        d = self.metadata.dependencies
+        if key in d:
+            result = d[key]
         return result
 
     def __repr__(self):
 
 
 class BaseInstalledDistribution(Distribution):
+
+    hasher = None
+
     def __init__(self, metadata, path, env=None):
         super(BaseInstalledDistribution, self).__init__(metadata)
         self.path = path
         self.dist_path  = env
 
+    def get_hash(self, data, hasher=None):
+        """
+        Get the hash of some data, using a particular hash algorithm, if
+        specified.
+
+        :param data: The data to be hashed.
+        :type data: bytes
+        :param hasher: The name of a hash implementation, supported by hashlib,
+                       or ``None``. Examples of valid values are ``'sha1'``,
+                       ``'sha224'``, ``'sha384'``, '``sha256'``, ``'md5'`` and
+                       ``'sha512'``. If no hasher is specified, the ``hasher``
+                       attribute of the :class:`InstalledDistribution` instance
+                       is used. If the hasher is determined to be ``None``, MD5
+                       is used as the hashing algorithm.
+        :returns: The hash of the data. If a hasher was explicitly specified,
+                  the returned hash will be prefixed with the specified hasher
+                  followed by '='.
+        :rtype: str
+        """
+        if hasher is None:
+            hasher = self.hasher
+        if hasher is None:
+            hasher = hashlib.md5
+            prefix = ''
+        else:
+            hasher = getattr(hashlib, hasher)
+            prefix = '%s=' % self.hasher
+        return '%s%s' % (prefix, hasher(data).hexdigest())
+
 class InstalledDistribution(BaseInstalledDistribution):
     """Created with the *path* of the ``.dist-info`` directory provided to the
     constructor. It reads the metadata contained in ``METADATA`` when it is
     present (in other words, whether the package was installed by user
     request or it was installed as a dependency)."""
 
-    hasher = None
-
     def __init__(self, path, env=None):
         if env and env._cache_enabled and path in env._cache.path:
             metadata = env._cache.path[path].metadata
         for result in self._get_records(local):
             yield result
 
-    def get_hash(self, data, hasher=None):
-        """
-        Get the hash of some data, using a particular hash algorithm, if
-        specified.
-
-        :param data: The data to be hashed.
-        :type data: bytes
-        :param hasher: The name of a hash implementation, supported by hashlib,
-                       or ``None``. Examples of valid values are ``'sha1'``,
-                       ``'sha224'``, ``'sha384'``, '``sha256'``, ``'md5'`` and
-                       ``'sha512'``. If no hasher is specified, the ``hasher``
-                       attribute of the :class:`InstalledDistribution` instance
-                       is used. If the hasher is determined to be ``None``, MD5
-                       is used as the hashing algorithm.
-        :returns: The hash of the data. If a hasher was explicitly specified,
-                  the returned hash will be prefixed with the specified hasher
-                  followed by '='.
-        :rtype: str
-        """
-        if hasher is None:
-            hasher = self.hasher
-        if hasher is None:
-            hasher = hashlib.md5
-            prefix = ''
-        else:
-            hasher = getattr(hashlib, hasher)
-            prefix = '%s=' % self.hasher
-        return '%s%s' % (prefix, hasher(data).hexdigest())
-
     def write_installed_files(self, paths):
         """
         Writes the ``RECORD`` file, using the ``paths`` iterable passed in. Any
     def __init__(self, path, env=None):
         self.path = path
         self.dist_path  = env
-        if env._cache_enabled and path in env._cache_egg.path:
+        if env and env._cache_enabled and path in env._cache_egg.path:
             metadata = env._cache_egg.path[path].metadata
             self.name = metadata['Name']
             self.version = metadata['Version']
     def __str__(self):
         return "%s %s" % (self.name, self.version)
 
+    def check_installed_files(self):
+        """
+        Checks that the hashes and sizes of the files in ``RECORD`` are
+        matched by the files themselves. Returns a (possibly empty) list of
+        mismatches. Each entry in the mismatch list will be a tuple consisting
+        of the path, 'exists', 'size' or 'hash' according to what didn't match
+        (existence is checked first, then size, then hash), the expected
+        value and the actual value.
+        """
+        mismatches = []
+        record_path = os.path.join(self.path, 'installed-files.txt')
+        if os.path.exists(record_path):
+            for path, hash, size in self.list_installed_files():
+                if path == record_path:
+                    continue
+                if not os.path.exists(path):
+                    mismatches.append((path, 'exists', True, False))
+        return mismatches
+
     def list_installed_files(self, local=False):
 
         def _md5(path):
                 content = f.read()
             finally:
                 f.close()
-            return md5(content).hexdigest()
+            return hashlib.md5(content).hexdigest()
 
         def _size(path):
             return os.stat(path).st_size
 
     # now make the edges
     for dist in dists:
+        # need to leave this in because tests currently rely on it ...
         requires = dist.metadata['Requires-Dist'] + dist.metadata['Requires']
+        if not requires:
+            requires = dist.get_requirements('install')
         for req in requires:
             try:
                 matcher = scheme.matcher(req)

distlib/locators.py

                 md['Version'] = version = info['version']
                 md['Download-URL'] = info['url']
                 dist = Distribution(md)
+                md.dependencies = info.get('requirements', {})
                 result[version] = dist
         return result
 

distlib/metadata.py

         self.platform_dependent = platform_dependent
         self.execution_context = execution_context
         self.scheme = get_scheme('default')
+        self.dependencies = {}
         if [path, fileobj, mapping].count(None) < 2:
             raise TypeError('path, fileobj and mapping are exclusive')
         if path is not None:
     return result
 
 
-def get_extended_metadata(name, version):
-    url = ('http://www.red-dove.com/pypi/results/'
-           '%s/%s/%s/info/package.json' % (name[0].upper(), name,
-                                           version))
-    return _get_external_data(url)
-
-
 def get_release_data(name):
     url = ('http://www.red-dove.com/pypi/projects/'
            '%s/%s/project.json' % (name[0].upper(), name))

tests/distlib_tests.py

 from test_markers import MarkersTestCase
 from test_metadata import MetadataTestCase
 from test_database import (DataFilesTestCase, TestDatabase, TestDistribution,
-                           DepGraphTestCase)
+                           TestEggInfoDistribution, DepGraphTestCase)
 from test_resources import (ZipResourceTestCase, FileResourceTestCase,
                             CacheTestCase)
 from test_scripts import ScriptTestCase

tests/fake_dists/bacon-0.1.egg-info/installed-files.txt

+../dummy.py
+../dummy.pyc
+./
+PKG-INFO
+SOURCES.txt
+top_level.txt
+dependency_links.txt

tests/test_database.py

                 self.assertEqual(hash, record_data[path][0])
                 self.assertEqual(size, record_data[path][1])
 
-    def test_check_installed_files(self):
-        for dir_ in self.dirs:
-            dist = self.cls(dir_)
-            mismatches = dist.check_installed_files()
-            self.assertEqual(mismatches, [])
-            # pick a non-empty file at random and change its contents
-            # but not its size. Check the failure returned,
-            # then restore the file.
-            files = [f for f in dist.list_installed_files() if f[-1] not in ('', '0')]
-            bad_file = random.choice(files)
-            bad_file_name = bad_file[0]
-            with open(bad_file_name, 'rb') as f:
-                data = f.read()
-            bad_data = bytes(bytearray(reversed(data)))
-            bad_hash = dist.get_hash(bad_data)
-            with open(bad_file_name, 'wb') as f:
-                f.write(bad_data)
-            mismatches = dist.check_installed_files()
-            self.assertEqual(mismatches, [(bad_file_name, 'hash', bad_file[1],
-                                           bad_hash)])
-            # now truncate the file by one byte and see what's returned
-            with open(bad_file_name, 'wb') as f:
-                f.write(bad_data[:-1])
-            bad_size = str(len(bad_data) - 1)
-            mismatches = dist.check_installed_files()
-            self.assertEqual(mismatches, [(bad_file_name, 'size', bad_file[2],
-                                           bad_size)])
-
-            # now remove the file and see what's returned
-            os.remove(bad_file_name)
-            mismatches = dist.check_installed_files()
-            self.assertEqual(mismatches, [(bad_file_name, 'exists',
-                                           True, False)])
-
-            # restore the file
-            with open(bad_file_name, 'wb') as f:
-                f.write(data)
-
     def test_hash(self):
         datalen = random.randrange(0, 500)
         data = os.urandom(datalen)
         self.assertEqual(resource_path, 'babar.png')
         self.assertRaises(KeyError, dist.get_resource_path, 'notexist')
 
+    def test_check_installed_files(self):
+        for dir_ in self.dirs:
+            dist = self.cls(dir_)
+            mismatches = dist.check_installed_files()
+            self.assertEqual(mismatches, [])
+            # pick a non-empty file at random and change its contents
+            # but not its size. Check the failure returned,
+            # then restore the file.
+            files = [f for f in dist.list_installed_files() if f[-1] not in ('', '0')]
+            bad_file = random.choice(files)
+            bad_file_name = bad_file[0]
+            with open(bad_file_name, 'rb') as f:
+                data = f.read()
+            bad_data = bytes(bytearray(reversed(data)))
+            bad_hash = dist.get_hash(bad_data)
+            with open(bad_file_name, 'wb') as f:
+                f.write(bad_data)
+            mismatches = dist.check_installed_files()
+            self.assertEqual(mismatches, [(bad_file_name, 'hash', bad_file[1],
+                                           bad_hash)])
+            # now truncate the file by one byte and see what's returned
+            with open(bad_file_name, 'wb') as f:
+                f.write(bad_data[:-1])
+            bad_size = str(len(bad_data) - 1)
+            mismatches = dist.check_installed_files()
+            self.assertEqual(mismatches, [(bad_file_name, 'size', bad_file[2],
+                                           bad_size)])
+
+            # now remove the file and see what's returned
+            os.remove(bad_file_name)
+            mismatches = dist.check_installed_files()
+            self.assertEqual(mismatches, [(bad_file_name, 'exists',
+                                           True, False)])
+
+            # restore the file
+            with open(bad_file_name, 'wb') as f:
+                f.write(data)
+
 
 class TestEggInfoDistribution(CommonDistributionTests,
                               LoggingCatcher,
                      if f.endswith('.egg') or f.endswith('.egg-info')]
 
         self.records = {}
+        for egginfo_dir in self.dirs:
+            dist_location = egginfo_dir.replace('.egg-info', '')
+            record_file = os.path.join(egginfo_dir, 'installed-files.txt')
+
+            dist = self.cls(egginfo_dir)
+            #dist.write_installed_files(get_files(dist_location))
+
+            record_data = {}
+            if os.path.exists(record_file):
+                with open(record_file) as fp:
+                    for line in fp:
+                        line = line.strip()
+                        if line == './':
+                            break
+                        record_data[line] = None, None
+            self.records[egginfo_dir] = record_data
 
     @unittest.skip('not implemented yet')
     def test_list_installed_files(self):