Commits

AJ R  committed ee15fef

Many changes wrt SavWriter, setup.py and tox

  • Participants
  • Parent commits 8a0b6cf

Comments (0)

Files changed (10)

 
 This file can also be run as a script to install or upgrade setuptools.
 """
+import os
+import shutil
 import sys
-DEFAULT_VERSION = "0.6c11"
-DEFAULT_URL     = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3]
-
-md5_data = {
-    'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca',
-    'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb',
-    'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b',
-    'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a',
-    'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618',
-    'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac',
-    'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5',
-    'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4',
-    'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c',
-    'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b',
-    'setuptools-0.6c10-py2.3.egg': 'ce1e2ab5d3a0256456d9fc13800a7090',
-    'setuptools-0.6c10-py2.4.egg': '57d6d9d6e9b80772c59a53a8433a5dd4',
-    'setuptools-0.6c10-py2.5.egg': 'de46ac8b1c97c895572e5e8596aeb8c7',
-    'setuptools-0.6c10-py2.6.egg': '58ea40aef06da02ce641495523a0b7f5',
-    'setuptools-0.6c11-py2.3.egg': '2baeac6e13d414a9d28e7ba5b5a596de',
-    'setuptools-0.6c11-py2.4.egg': 'bd639f9b0eac4c42497034dec2ec0c2b',
-    'setuptools-0.6c11-py2.5.egg': '64c94f3bf7a72a13ec83e0b24f2749b2',
-    'setuptools-0.6c11-py2.6.egg': 'bfa92100bd772d5a213eedd356d64086',
-    'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27',
-    'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277',
-    'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa',
-    'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e',
-    'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e',
-    'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f',
-    'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2',
-    'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc',
-    'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167',
-    'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64',
-    'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d',
-    'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20',
-    'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab',
-    'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53',
-    'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2',
-    'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e',
-    'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372',
-    'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902',
-    'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de',
-    'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b',
-    'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03',
-    'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a',
-    'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6',
-    'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a',
-}
-
-import sys, os
-try: from hashlib import md5
-except ImportError: from md5 import md5
-
-def _validate_md5(egg_name, data):
-    if egg_name in md5_data:
-        digest = md5(data).hexdigest()
-        if digest != md5_data[egg_name]:
-            print >>sys.stderr, (
-                "md5 validation of %s failed!  (Possible download problem?)"
-                % egg_name
-            )
-            sys.exit(2)
-    return data
-
-def use_setuptools(
-    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
-    download_delay=15
-):
-    """Automatically find/download setuptools and make it available on sys.path
-
-    `version` should be a valid setuptools version number that is available
-    as an egg for download under the `download_base` URL (which should end with
-    a '/').  `to_dir` is the directory where setuptools will be downloaded, if
-    it is not already available.  If `download_delay` is specified, it should
-    be the number of seconds that will be paused before initiating a download,
-    should one be required.  If an older version of setuptools is installed,
-    this routine will print a message to ``sys.stderr`` and raise SystemExit in
-    an attempt to abort the calling script.
-    """
-    was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules
-    def do_download():
-        egg = download_setuptools(version, download_base, to_dir, download_delay)
-        sys.path.insert(0, egg)
-        import setuptools; setuptools.bootstrap_install_from = egg
+import tempfile
+import tarfile
+import optparse
+import subprocess
+
+from distutils import log
+
+try:
+    from site import USER_SITE
+except ImportError:
+    USER_SITE = None
+
+DEFAULT_VERSION = "0.9.6"
+DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
+
+def _python_cmd(*args):
+    args = (sys.executable,) + args
+    return subprocess.call(args) == 0
+
+def _install(tarball, install_args=()):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # installing
+        log.warn('Installing Setuptools')
+        if not _python_cmd('setup.py', 'install', *install_args):
+            log.warn('Something went wrong during the installation.')
+            log.warn('See the error message above.')
+            # exitcode will be 2
+            return 2
+    finally:
+        os.chdir(old_wd)
+        shutil.rmtree(tmpdir)
+
+
+def _build_egg(egg, tarball, to_dir):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # building an egg
+        log.warn('Building a Setuptools egg in %s', to_dir)
+        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
+
+    finally:
+        os.chdir(old_wd)
+        shutil.rmtree(tmpdir)
+    # returning the result
+    log.warn(egg)
+    if not os.path.exists(egg):
+        raise IOError('Could not build the egg.')
+
+
+def _do_download(version, download_base, to_dir, download_delay):
+    egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg'
+                       % (version, sys.version_info[0], sys.version_info[1]))
+    if not os.path.exists(egg):
+        tarball = download_setuptools(version, download_base,
+                                      to_dir, download_delay)
+        _build_egg(egg, tarball, to_dir)
+    sys.path.insert(0, egg)
+    import setuptools
+    setuptools.bootstrap_install_from = egg
+
+
+def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                   to_dir=os.curdir, download_delay=15):
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    was_imported = 'pkg_resources' in sys.modules or \
+        'setuptools' in sys.modules
     try:
         import pkg_resources
     except ImportError:
-        return do_download()       
+        return _do_download(version, download_base, to_dir, download_delay)
     try:
-        pkg_resources.require("setuptools>="+version); return
-    except pkg_resources.VersionConflict, e:
+        pkg_resources.require("setuptools>=" + version)
+        return
+    except pkg_resources.VersionConflict:
+        e = sys.exc_info()[1]
         if was_imported:
-            print >>sys.stderr, (
-            "The required version of setuptools (>=%s) is not available, and\n"
-            "can't be installed while this script is running. Please install\n"
-            " a more recent version first, using 'easy_install -U setuptools'."
-            "\n\n(Currently using %r)"
-            ) % (version, e.args[0])
+            sys.stderr.write(
+            "The required version of setuptools (>=%s) is not available,\n"
+            "and can't be installed while this script is running. Please\n"
+            "install a more recent version first, using\n"
+            "'easy_install -U setuptools'."
+            "\n\n(Currently using %r)\n" % (version, e.args[0]))
             sys.exit(2)
+        else:
+            del pkg_resources, sys.modules['pkg_resources']    # reload ok
+            return _do_download(version, download_base, to_dir,
+                                download_delay)
     except pkg_resources.DistributionNotFound:
-        pass
+        return _do_download(version, download_base, to_dir,
+                            download_delay)
 
-    del pkg_resources, sys.modules['pkg_resources']    # reload ok
-    return do_download()
 
-def download_setuptools(
-    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
-    delay = 15
-):
+def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                        to_dir=os.curdir, delay=15):
     """Download setuptools from a specified location and return its filename
 
     `version` should be a valid setuptools version number that is available
     as an egg for download under the `download_base` URL (which should end
     with a '/'). `to_dir` is the directory where the egg will be downloaded.
-    `delay` is the number of seconds to pause before an actual download attempt.
+    `delay` is the number of seconds to pause before an actual download
+    attempt.
     """
-    import urllib2, shutil
-    egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
-    url = download_base + egg_name
-    saveto = os.path.join(to_dir, egg_name)
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+    tgz_name = "setuptools-%s.tar.gz" % version
+    url = download_base + tgz_name
+    saveto = os.path.join(to_dir, tgz_name)
     src = dst = None
     if not os.path.exists(saveto):  # Avoid repeated downloads
         try:
-            from distutils import log
-            if delay:
-                log.warn("""
----------------------------------------------------------------------------
-This script requires setuptools version %s to run (even to display
-help).  I will attempt to download it for you (from
-%s), but
-you may need to enable firewall access for this script first.
-I will start the download in %d seconds.
-
-(Note: if this machine does not have network access, please obtain the file
-
-   %s
-
-and place it in this directory before rerunning this script.)
----------------------------------------------------------------------------""",
-                    version, download_base, delay, url
-                ); from time import sleep; sleep(delay)
             log.warn("Downloading %s", url)
-            src = urllib2.urlopen(url)
+            src = urlopen(url)
             # Read/write all in one block, so we don't create a corrupt file
             # if the download is interrupted.
-            data = _validate_md5(egg_name, src.read())
-            dst = open(saveto,"wb"); dst.write(data)
+            data = src.read()
+            dst = open(saveto, "wb")
+            dst.write(data)
         finally:
-            if src: src.close()
-            if dst: dst.close()
+            if src:
+                src.close()
+            if dst:
+                dst.close()
     return os.path.realpath(saveto)
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-def main(argv, version=DEFAULT_VERSION):
-    """Install or upgrade setuptools and EasyInstall"""
-    try:
-        import setuptools
-    except ImportError:
-        egg = None
-        try:
-            egg = download_setuptools(version, delay=0)
-            sys.path.insert(0,egg)
-            from setuptools.command.easy_install import main
-            return main(list(argv)+[egg])   # we're done here
-        finally:
-            if egg and os.path.exists(egg):
-                os.unlink(egg)
+def _extractall(self, path=".", members=None):
+    """Extract all members from the archive to the current working
+       directory and set owner, modification time and permissions on
+       directories afterwards. `path' specifies a different directory
+       to extract to. `members' is optional and must be a subset of the
+       list returned by getmembers().
+    """
+    import copy
+    import operator
+    from tarfile import ExtractError
+    directories = []
+
+    if members is None:
+        members = self
+
+    for tarinfo in members:
+        if tarinfo.isdir():
+            # Extract directories with a safe mode.
+            directories.append(tarinfo)
+            tarinfo = copy.copy(tarinfo)
+            tarinfo.mode = 448  # decimal for oct 0700
+        self.extract(tarinfo, path)
+
+    # Reverse sort directories.
+    if sys.version_info < (2, 4):
+        def sorter(dir1, dir2):
+            return cmp(dir1.name, dir2.name)
+        directories.sort(sorter)
+        directories.reverse()
     else:
-        if setuptools.__version__ == '0.0.1':
-            print >>sys.stderr, (
-            "You have an obsolete version of setuptools installed.  Please\n"
-            "remove it from your system entirely before rerunning this script."
-            )
-            sys.exit(2)
+        directories.sort(key=operator.attrgetter('name'), reverse=True)
 
-    req = "setuptools>="+version
-    import pkg_resources
-    try:
-        pkg_resources.require(req)
-    except pkg_resources.VersionConflict:
+    # Set correct owner, mtime and filemode on directories.
+    for tarinfo in directories:
+        dirpath = os.path.join(path, tarinfo.name)
         try:
-            from setuptools.command.easy_install import main
-        except ImportError:
-            from easy_install import main
-        main(list(argv)+[download_setuptools(delay=0)])
-        sys.exit(0) # try to force an exit
-    else:
-        if argv:
-            from setuptools.command.easy_install import main
-            main(argv)
-        else:
-            print "Setuptools version",version,"or greater has been installed."
-            print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)'
-
-def update_md5(filenames):
-    """Update our built-in md5 registry"""
-
-    import re
-
-    for name in filenames:
-        base = os.path.basename(name)
-        f = open(name,'rb')
-        md5_data[base] = md5(f.read()).hexdigest()
-        f.close()
-
-    data = ["    %r: %r,\n" % it for it in md5_data.items()]
-    data.sort()
-    repl = "".join(data)
-
-    import inspect
-    srcfile = inspect.getsourcefile(sys.modules[__name__])
-    f = open(srcfile, 'rb'); src = f.read(); f.close()
-
-    match = re.search("\nmd5_data = {\n([^}]+)}", src)
-    if not match:
-        print >>sys.stderr, "Internal error!"
-        sys.exit(2)
-
-    src = src[:match.start(1)] + repl + src[match.end(1):]
-    f = open(srcfile,'w')
-    f.write(src)
-    f.close()
-
-
-if __name__=='__main__':
-    if len(sys.argv)>2 and sys.argv[1]=='--md5update':
-        update_md5(sys.argv[2:])
-    else:
-        main(sys.argv[1:])
-
-
-
-
-
+            self.chown(tarinfo, dirpath)
+            self.utime(tarinfo, dirpath)
+            self.chmod(tarinfo, dirpath)
+        except ExtractError:
+            e = sys.exc_info()[1]
+            if self.errorlevel > 1:
+                raise
+            else:
+                self._dbg(1, "tarfile: %s" % e)
+
+
+def _build_install_args(options):
+    """
+    Build the arguments to 'python setup.py install' on the setuptools package
+    """
+    install_args = []
+    if options.user_install:
+        if sys.version_info < (2, 6):
+            log.warn("--user requires Python 2.6 or later")
+            raise SystemExit(1)
+        install_args.append('--user')
+    return install_args
+
+def _parse_args():
+    """
+    Parse the command line for options
+    """
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '--user', dest='user_install', action='store_true', default=False,
+        help='install in user site package (requires Python 2.6 or later)')
+    parser.add_option(
+        '--download-base', dest='download_base', metavar="URL",
+        default=DEFAULT_URL,
+        help='alternative URL from where to download the setuptools package')
+    options, args = parser.parse_args()
+    # positional arguments are ignored
+    return options
+
+def main(version=DEFAULT_VERSION):
+    """Install or upgrade setuptools and EasyInstall"""
+    options = _parse_args()
+    tarball = download_setuptools(download_base=options.download_base)
+    return _install(tarball, _build_install_args(options))
 
+if __name__ == '__main__':
+    sys.exit(main())

File savReaderWriter/generic.py

 import collections
 
 from savReaderWriter import *
+from py3k import *
 
 class Generic(object):
     """
         fdopen.argtypes = [c_int, c_char_p]
         fdopen.restype = c_void_p
         fdopen.errcheck = self.errcheck
-        mode_ = b"wb" if mode == b"cp" else bytes(mode)
+        mode_ = b"wb" if mode == b"cp" else bytez(mode)
         with open(savFileName, mode_.decode("utf-8")) as f:
             self.fd = fdopen(f.fileno(), mode_)
         if mode == "rb":

File savReaderWriter/header.py

         """Decorator to Utf-8 decode all str items contained in a dictionary
         If ioUtf8=True, the dictionary's keys and values are decoded, but only
         values that are strs, lists, or dicts."""
-        bytes_ = __builtins__["bytes"] if sys.version_info[0] > 2 else str  # TODO
+        bytes_ = bytes if sys.version_info[0] > 2 else str
         uS = lambda x: x.decode("utf-8") if isinstance(x, bytes_) else x
         uL = lambda x: map(uS, x) if isinstance(x, list) else x
         @functools.wraps(func)
 
             printFormat = allFormats.get(printFormat_.value)[0]
             printFormat = printFormat.split(b"_")[-1]
-            format_ = printFormat + bytes(str(printWid_.value))
+            format_ = printFormat + bytez(str(printWid_.value))
             if self.varTypes[varName] == 0:
-                format_ += (b"." + bytes(str(printDec_.value)))
+                format_ += (b"." + bytez(str(printDec_.value)))
             if format_.endswith(b".0"):
                 format_ = format_[:-2]
             self.formats_[varName] = format_

File savReaderWriter/py3k.py

 
 # bytes keyword has a second argument in Python 3
 if isPy3k:
-    bytes = functools.partial(bytes, encoding="utf-8")
+    bytez = functools.partial(bytes, encoding="utf-8")
+else:
+    bytez = bytes
+
+def bytify(encoding):
+    if isPy3k:
+        def func(value):
+            return bytes(value, encoding)
+    else:
+        def func(value):
+            return bytes(value)
+    func.__doc__ = "bytes wrapper for python 2 and 3"
+    return func
 
 # ctypes.c_char_p does not take unicode strings
 if isPy3k:

File savReaderWriter/savHeaderReader.py

                             report.append("%s -- %s" % (varName, values))
             else:
                 # varname, file label
-                bytes = __builtins__.bytes  # TODO shadowed-->functools.partial 
                 if isinstance(allValues, (str, bytes, unicode)) and allValues:
                     allValues = [allValues]
                 for varName in allValues:

File savReaderWriter/savReader.py

                 self.gregorianEpoch = datetime.datetime(1582, 10, 14, 0, 0, 0)
             theDate = (self.gregorianEpoch +
                        datetime.timedelta(seconds=spssDateValue))
-            return bytes(datetime.datetime.strftime(theDate, fmt))
+            return bytez(datetime.datetime.strftime(theDate, fmt))
         except (OverflowError, TypeError, ValueError):
             return recodeSysmisTo
 

File savReaderWriter/savWriter.py

 import os
 import time
 
-
+from savReaderWriter import *
+from py3k import *
 from header import *
 if cWriterowOK:
     cWriterow = cWriterow.cWriterow
         self.sysmis_ = self.sysmis
         self.ioUtf8_ = ioUtf8
         self.pad_8_lookup = self._getPaddingLookupTable(self.varTypes)
+        self.bytify = bytify(self.fileEncoding)  # from py3k module
 
         if self.mode == "wb":
             self._openWrite(self.savFileName, self.overwrite)
         {1:%-8s, 7:%-8s, 9: %-16s, 24: %-24s}. Purpose: Get rid of trailing
         null bytes"""
         strLengths = varTypes.values()
-        return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strLengths])
+        lookup = dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strLengths])
+        #if sys.version_info[1] > 2:
+        #    return {k: bytes(v, "utf-8") for k, v in lookup.items()}
+        return lookup
 
     def writerow(self, record):
         """ This function writes one record, which is a Python list."""
         """ This function writes one record, which is a Python list,
         compare this Python version with the Cython version cWriterow."""
         float_ = float
+        bytify = self.bytify
         for i, value in enumerate(record):
             varName = self.varNames[i]
             varType = self.varTypes[varName]
                     value = self.sysmis_
             else:
                 # Get rid of trailing null bytes --> 7 x faster than 'ljust'
-                value = bytes(self.pad_8_lookup[varType] % value)
+                value = bytify(self.pad_8_lookup[varType] % value)
                 if self.ioUtf8_:
                     if isinstance(value, unicode):
                         value = value.encode("utf-8")

File savReaderWriter/unit_tests/test_SavWriter_ioLocale.py

         self.savFileName = os.path.join(tempfile.gettempdir(),
                                         "german_out.sav")
         self.kwargs = dict(savFileName=self.savFileName, 
-                           varNames=['\xfcberhaupt'],
-                           varTypes={'\xfcberhaupt': 0},
+                           varNames=[b'\xfcberhaupt'],
+                           varTypes={b'\xfcberhaupt': 0},
                            ioLocale=germanlocale)
 
     @unittest.skipUnless(bool(germanlocale), "German locale not present")
                       ioLocale=germanlocale)
         with SavReader(**kwargs) as reader:
             varNames = next(reader)
-            self.assertEqual(varNames, ['\xfcberhaupt'])
+            self.assertEqual(varNames, [b'\xfcberhaupt'])
 
     def tearDown(self):
         locale.setlocale(locale.LC_ALL, oldLocale) 
 import platform
 
 sys.path.append(os.path.dirname(os.path.realpath(__file__)))
-from ez_setup import use_setuptools
-use_setuptools()
+try:
+    from ez_setup import use_setuptools
+    use_setuptools()
+except:
+    pass # Tox
 from setuptools import setup
 
 def read(fname):
 [tox]
-envlist = py27,py33
+envlist = py27,py33,pypy
+indexserver = 
+    default = https://pypi.python.org/pypi
 [testenv]
 commands = nosetests
+deps = setuptools