Commits

holger krekel committed efe7d94

move all interpreter information detection to tox/interpreters.py

Comments (0)

Files changed (9)

   installation command with options for dep/pkg install.  Thanks Carl Meyer
   for the PR and docs.
 
-
 - address issueintroduce python2.5 support by vendoring the virtualenv-1.9.1 script
   and forcing pip<1.4. Also the default [py25] environment modifies the
   default installer_command (new config option) to use pip without the "--pre" 
 
 - if a HOMEDIR cannot be determined, use the toxinidir.
 
+- refactor interpreter information detection to live in new
+  tox/interpreters.py file, tests in tests/test_interpreters.py.
+
 1.5.0
 -----------------
 
     and any defined dependencies. Must contain the substitution key
     ``{packages}`` which will be replaced by the packages to
     install. May also contain the substitution key ``{opts}``, which
-    will be replaced by the ``-i`` option to specify index server
-    (according to :confval:`indexserver` and the ``:indexserver:dep``
+    will be replaced by the ``-i INDEXURL`` option if an index server
+    is active (see :confval:`indexserver` and the ``:indexserver:dep``
     syntax of :confval:`deps`) and the ``--download-cache`` option (if
     you've specified :confval:`downloadcache`). If your installer does
     not support ``-i`` and ``--download-cache`` command-line options,

tests/test_config.py

 from textwrap import dedent
 
 import py
-from tox._config import IniReader, CommandParser
-from tox._config import parseconfig
-from tox._config import prepare_parse, _split_env
+from tox._config import *
+from tox._config import _split_env
+
 
 class TestVenvConfig:
     def test_config_parsing_minimal(self, tmpdir, newconfig):
         assert envconfig.changedir.basename == "abc"
         assert envconfig.changedir == config.setupdir.join("abc")
 
-    def test_install_command_defaults_py25(self, newconfig):
+    def test_install_command_defaults_py25(self, newconfig, monkeypatch):
+        from tox.interpreters import Interpreters
+        def get_info(self, name):
+            if "x25" in name:
+                class I:
+                    runnable = True
+                    executable = "python2.5"
+                    version_info = (2,5)
+            else:
+                class I:
+                    runnable = False
+                    executable = "python"
+            return I
+        monkeypatch.setattr(Interpreters, "get_info", get_info)
         config = newconfig("""
-            [testenv:py25]
+            [testenv:x25]
+            basepython = x25
             [testenv:py25-x]
+            basepython = x25
             [testenv:py26]
+            basepython = "python"
         """)
-        for name in ("py25", "py25-x"):
+        for name in ("x25", "py25-x"):
             env = config.envconfigs[name]
             assert env.install_command == \
                "pip install --insecure {opts} {packages}".split()
         assert conf.changedir.basename == 'testing'
         assert conf.changedir.dirpath().realpath() == tmpdir.realpath()
 
-    @pytest.mark.xfailif("sys.platform == 'win32'")
-    def test_substitution_envsitepackagesdir(self, tmpdir, monkeypatch,
-                                             newconfig):
-        """
-         The envsitepackagesdir property is mostly doing system work,
-         so this test doesn't excercise it very well.
-
-         Usage of envsitepackagesdir on win32/jython will explicitly
-         throw an exception,
-        """
-        class MockPopen(object):
-            returncode = 0
-
-            def __init__(self, *args, **kwargs):
-                pass
-
-            def communicate(self, *args, **kwargs):
-                return 'onevalue', 'othervalue'
-
-        monkeypatch.setattr(subprocess, 'Popen', MockPopen)
-        env = 'py%s' % (''.join(sys.version.split('.')[0:2]))
-        config = newconfig("""
-            [testenv:%s]
-            commands = {envsitepackagesdir}
-        """ % (env))
-        conf = config.envconfigs[env]
-        argv = conf.commands
-        assert argv[0][0] == 'onevalue'
-
-
 class TestGlobalOptions:
     def test_notest(self, newconfig):
         config = newconfig([], "")

tests/test_interpreters.py

+import sys
+import os
+
+import pytest
+from tox.interpreters import *
+
+@pytest.fixture
+def interpreters():
+    return Interpreters()
+
+@pytest.mark.skipif("sys.platform != 'win32'")
+def test_locate_via_py(monkeypatch):
+    from tox._venv import locate_via_py
+    class PseudoPy:
+        def sysexec(self, *args):
+            assert args[0] == '-3.2'
+            assert args[1] == '-c'
+            # Return value needs to actually exist!
+            return sys.executable
+    @staticmethod
+    def ret_pseudopy(name):
+        assert name == 'py'
+        return PseudoPy()
+    # Monkeypatch py.path.local.sysfind to return PseudoPy
+    monkeypatch.setattr(py.path.local, 'sysfind', ret_pseudopy)
+    assert locate_via_py('3', '2') == sys.executable
+
+def test_find_executable():
+    p = find_executable(sys.executable)
+    assert p == py.path.local(sys.executable)
+    for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split():
+        name = "python%s" % ver
+        if sys.platform == "win32":
+            pydir = "python%s" % ver.replace(".", "")
+            x = py.path.local("c:\%s" % pydir)
+            print (x)
+            if not x.check():
+                continue
+        else:
+            if not py.path.local.sysfind(name):
+                continue
+        p = find_executable(name)
+        assert p
+        popen = py.std.subprocess.Popen([str(p), '-V'],
+                stderr=py.std.subprocess.PIPE)
+        stdout, stderr = popen.communicate()
+        assert ver in py.builtin._totext(stderr, "ascii")
+
+def test_find_executable_extra(monkeypatch):
+    @staticmethod
+    def sysfind(x):
+        return "hello"
+    monkeypatch.setattr(py.path.local, "sysfind", sysfind)
+    t = find_executable("qweqwe")
+    assert t == "hello"
+
+def test_run_and_get_interpreter_info():
+    name = os.path.basename(sys.executable)
+    info = run_and_get_interpreter_info(name, sys.executable)
+    assert info.version_info == tuple(sys.version_info)
+    assert info.name == name
+    assert info.executable == sys.executable
+
+class TestInterpreters:
+
+    def test_get_info_self_exceptions(self, interpreters):
+        pytest.raises(ValueError, lambda:
+                      interpreters.get_info())
+        pytest.raises(ValueError, lambda:
+                      interpreters.get_info(name="12", executable="123"))
+
+    def test_get_executable(self, interpreters):
+        x = interpreters.get_executable(sys.executable)
+        assert x == sys.executable
+        assert not interpreters.get_executable("12l3k1j23")
+
+    def test_get_info__name(self, interpreters):
+        basename = os.path.basename(sys.executable)
+        info = interpreters.get_info(basename)
+        assert info.version_info == tuple(sys.version_info)
+        assert info.name == basename
+        assert info.executable == sys.executable
+        assert info.runnable
+
+    def test_get_info__name_not_exists(self, interpreters):
+        info = interpreters.get_info("qlwkejqwe")
+        assert not info.version_info
+        assert info.name == "qlwkejqwe"
+        assert not info.executable
+        assert not info.runnable
+
+    def test_get_sitepackagesdir_error(self, interpreters):
+        info = interpreters.get_info(sys.executable)
+        s = interpreters.get_sitepackagesdir(info, "")
+        assert s

tests/test_venv.py

 import tox
 import pytest
 import os, sys
-from tox._venv import VirtualEnv, CreationConfig, getdigest
-from tox._venv import find_executable
-from tox._venv import _getinterpreterversion
+from tox._venv import *
 
 py25calls = int(sys.version_info[:2] == (2,5))
 
 def test_getdigest(tmpdir):
     assert getdigest(tmpdir) == "0"*32
 
-@pytest.mark.skipif("sys.platform != 'win32'")
-def test_locate_via_py(monkeypatch):
-    from tox._venv import locate_via_py
-    class PseudoPy:
-        def sysexec(self, *args):
-            assert args[0] == '-3.2'
-            assert args[1] == '-c'
-            # Return value needs to actually exist!
-            return sys.executable
-    @staticmethod
-    def ret_pseudopy(name):
-        assert name == 'py'
-        return PseudoPy()
-    # Monkeypatch py.path.local.sysfind to return PseudoPy
-    monkeypatch.setattr(py.path.local, 'sysfind', ret_pseudopy)
-    assert locate_via_py('3', '2') == sys.executable
-
-def test_find_executable():
-    p = find_executable(sys.executable)
-    assert p == py.path.local(sys.executable)
-    for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split():
-        name = "python%s" % ver
-        if sys.platform == "win32":
-            pydir = "python%s" % ver.replace(".", "")
-            x = py.path.local("c:\%s" % pydir)
-            print (x)
-            if not x.check():
-                continue
-        else:
-            if not py.path.local.sysfind(name):
-                continue
-        p = find_executable(name)
-        assert p
-        popen = py.std.subprocess.Popen([str(p), '-V'],
-                stderr=py.std.subprocess.PIPE)
-        stdout, stderr = popen.communicate()
-        assert ver in py.builtin._totext(stderr, "ascii")
-
-def test_find_executable_extra(monkeypatch):
-    @staticmethod
-    def sysfind(x):
-        return "hello"
-    monkeypatch.setattr(py.path.local, "sysfind", sysfind)
-    t = find_executable("qweqwe")
-    assert t == "hello"
-
 def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession):
     config = newconfig([], """
         [testenv:python]
     py.test.raises(tox.exception.InterpreterNotFound,
                    venv.getsupportedinterpreter)
 
-def test_getinterpreterversion():
-    from distutils.sysconfig import get_python_version
-    version = _getinterpreterversion(sys.executable)
-    assert version == get_python_version()
 
 def test_create(monkeypatch, mocksession, newconfig):
     config = newconfig([], """
         #assert Envconfig.toxworkdir in args
         assert venv.getcommandpath("easy_install", cwd=py.path.local())
     interp = venv._getliveconfig().python
-    assert interp == venv.getconfigexecutable()
+    assert interp == venv.envconfig._basepython_info.executable
     assert venv.path_config.check(exists=False)
 
 @pytest.mark.skipif("sys.platform == 'win32'")
     # two different index servers, two calls
     assert len(l) == 3
     args = " ".join(l[0].args)
-    assert "-i" not in args
+    assert "-i " not in args
     assert "dep1" in args
 
     args = " ".join(l[1].args)
         for envconfig in self.config.envconfigs.values():
             self.report.line("[testenv:%s]" % envconfig.envname, bold=True)
             self.report.line("  basepython=%s" % envconfig.basepython)
+            self.report.line("  _basepython_info=%s" %
+                             envconfig._basepython_info)
             self.report.line("  envpython=%s" % envconfig.envpython)
             self.report.line("  envtmpdir=%s" % envconfig.envtmpdir)
             self.report.line("  envbindir=%s" % envconfig.envbindir)
 import subprocess
 import textwrap
 
+from tox.interpreters import Interpreters
+
 import py
 
 import tox
     def __init__(self):
         self.envconfigs = {}
         self.invocationcwd = py.path.local()
+        self.interpreters = Interpreters()
 
 class VenvConfig:
     def __init__(self, **kw):
 
     # no @property to avoid early calling (see callable(subst[key]) checks)
     def envsitepackagesdir(self):
-        print_envsitepackagesdir = textwrap.dedent("""
-        import sys
-        from distutils.sysconfig import get_python_lib
-        sys.stdout.write(get_python_lib(prefix=sys.argv[1]))
-        """)
-
-        exe = self.getsupportedinterpreter()
-        # can't use check_output until py27
-        proc = subprocess.Popen(
-            [str(exe), '-c', print_envsitepackagesdir, str(self.envdir)],
-            stdout=subprocess.PIPE)
-        odata, edata = proc.communicate()
-        if proc.returncode:
-            raise tox.exception.UnsupportedInterpreter(
-                "Error getting site-packages from %s" % self.basepython)
-        return odata
-
-    def getconfigexecutable(self):
-        from tox._venv import find_executable
-
-        python = self.basepython
-        if not python:
-            python = sys.executable
-        x = find_executable(str(python))
-        if x:
-            x = x.realpath()
+        self.getsupportedinterpreter()  # for throwing exceptions
+        x = self.config.interpreters.get_sitepackagesdir(
+                info=self._basepython_info,
+                envdir=self.envdir)
         return x
 
     def getsupportedinterpreter(self):
                 "jython" in self.basepython:
             raise tox.exception.UnsupportedInterpreter(
                 "Jython/Windows does not support installing scripts")
-        config_executable = self.getconfigexecutable()
-        if not config_executable:
+        info = self.config.interpreters.get_info(self.basepython)
+        if not info.executable:
             raise tox.exception.InterpreterNotFound(self.basepython)
-        return config_executable
+        return info.executable
+
 testenvprefix = "testenv:"
 
 class parseini:
         else:
             bp = sys.executable
         vc.basepython = reader.getdefault(section, "basepython", bp)
+        vc._basepython_info = config.interpreters.get_info(vc.basepython)
         reader.addsubstitions(envdir=vc.envdir, envname=vc.envname,
                               envbindir=vc.envbindir, envpython=vc.envpython,
                               envsitepackagesdir=vc.envsitepackagesdir)
         # need to use --insecure for pip commands because python2.5
         # doesn't support SSL
         pip_default_opts = ["{opts}", "{packages}"]
-        if "py25" in vc.envname:  # XXX too rough check for "python2.5"
+        info = vc._basepython_info
+        if info.runnable and info.version_info < (2,6):
             pip_default_opts.insert(0, "--insecure")
         else:
             pip_default_opts.insert(0, "--pre")
             return "could not install deps %s" %(self.envconfig.deps,)
 
     def _getliveconfig(self):
-        python = self.getconfigexecutable()
+        python = self.envconfig._basepython_info.executable
         md5 = getdigest(python)
         version = tox.__version__
         distribute = self.envconfig.distribute
             l.append(dep)
         return l
 
-    def getconfigexecutable(self):
-        return self.envconfig.getconfigexecutable()
-
     def getsupportedinterpreter(self):
         return self.envconfig.getsupportedinterpreter()
 
         #    return
         if action is None:
             action = self.session.newaction(self, "create")
+
+        interpreters = self.envconfig.config.interpreters
         config_interpreter = self.getsupportedinterpreter()
-        config_interpreter_version = _getinterpreterversion(
-            config_interpreter)
-        use_venv191 = config_interpreter_version < '2.6'
-        use_pip13 = config_interpreter_version < '2.6'
+        info = interpreters.get_info(executable=config_interpreter)
+        use_venv191 = use_pip13 = info.version_info < (2,6)
         if not use_venv191:
             f, path, _ = py.std.imp.find_module("virtualenv")
             f.close()
         self.session.report.verbosity2("setting PATH=%s" % os.environ["PATH"])
         return oldPATH
 
-def _getinterpreterversion(executable):
-    print_python_version = (
-        'from distutils.sysconfig import get_python_version\n'
-        'print(get_python_version())\n')
-    proc = subprocess.Popen([str(executable), '-c', print_python_version],
-                            stdout=subprocess.PIPE)
-    odata, edata = proc.communicate()
-    if proc.returncode:
-        raise tox.exception.UnsupportedInterpreter(
-            "Error getting python version from %s" % executable)
-    if sys.version_info[0] == 3:
-        string = str
-    else:
-        string = lambda x, encoding: str(x)
-    return string(odata, 'ascii').strip()
 
 def getdigest(path):
     path = py.path.local(path)
         return "0" * 32
     return path.computehash()
 
-if sys.platform != "win32":
-    def find_executable(name):
-        return py.path.local.sysfind(name)
-
-else:
-    # Exceptions to the usual windows mapping
-    win32map = {
-            'python': sys.executable,
-            'jython': "c:\jython2.5.1\jython.bat",
-    }
-    def locate_via_py(v_maj, v_min):
-        ver = "-%s.%s" % (v_maj, v_min)
-        script = "import sys; print(sys.executable)"
-        py_exe = py.path.local.sysfind('py')
-        if py_exe:
-            try:
-                exe = py_exe.sysexec(ver, '-c', script).strip()
-            except py.process.cmdexec.Error:
-                exe = None
-            if exe:
-                exe = py.path.local(exe)
-                if exe.check():
-                    return exe
-
-    def find_executable(name):
-        p = py.path.local.sysfind(name)
-        if p:
-            return p
-        actual = None
-        # Is this a standard PythonX.Y name?
-        m = re.match(r"python(\d)\.(\d)", name)
-        if m:
-            # The standard names are in predictable places.
-            actual = r"c:\python%s%s\python.exe" % m.groups()
-        if not actual:
-            actual = win32map.get(name, None)
-        if actual:
-            actual = py.path.local(actual)
-            if actual.check():
-                return actual
-        # The standard executables can be found as a last resort via the
-        # Python launcher py.exe
-        if m:
-            locate_via_py(*m.groups())
-
 
 def hack_home_env(homedir, index_url=None):
     # XXX HACK (this could also live with tox itself, consider)

tox/interpreters.py

+import sys
+import os
+import py
+import subprocess
+import inspect
+
+class Interpreters:
+    def __init__(self):
+        self.name2executable = {}
+        self.executable2info = {}
+
+    def get_executable(self, name):
+        """ return path object to the executable for the given
+        name (e.g. python2.5, python2.7, python etc.)
+        if name is already an existing path, return name.
+        If an interpreter cannot be found, return None.
+        """
+        try:
+            return self.name2executable[name]
+        except KeyError:
+            self.name2executable[name] = e = find_executable(name)
+            return e
+
+    def get_info(self, name=None, executable=None):
+        if name is None and executable is None:
+            raise ValueError("need to specify name or executable")
+        if name:
+            if executable is not None:
+                raise ValueError("cannot specify both name, executable")
+            executable = self.get_executable(name)
+        if not executable:
+            return NoInterpreterInfo(name=name)
+        try:
+            return self.executable2info[executable]
+        except KeyError:
+            info = run_and_get_interpreter_info(name, executable)
+            self.executable2info[executable] = info
+            return info
+
+    def get_sitepackagesdir(self, info, envdir):
+        if not info.executable:
+            return ""
+        envdir = str(envdir)
+        try:
+            res = exec_on_interpreter(info.executable,
+                [inspect.getsource(sitepackagesdir),
+                 "print (sitepackagesdir(%r))" % envdir])
+        except ExecFailed:
+            val = sys.exc_info()[1]
+            print ("execution failed: %s -- %s" %(val.out, val.err))
+            return ""
+        else:
+            return res["dir"]
+
+def run_and_get_interpreter_info(name, executable):
+    assert executable
+    try:
+        result = exec_on_interpreter(executable,
+            [inspect.getsource(pyinfo), "print (pyinfo())"])
+    except ExecFailed:
+        val = sys.exc_info()[1]
+        return NoInterpreterInfo(name, **val.__dict__)
+    else:
+        return InterpreterInfo(name, executable, **result)
+
+def exec_on_interpreter(executable, source):
+    if isinstance(source, list):
+        source = "\n".join(source)
+    from subprocess import Popen, PIPE
+    args = [str(executable)]
+    popen = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    popen.stdin.write(source.encode("utf8"))
+    out, err = popen.communicate()
+    if popen.returncode:
+        raise ExecFailed(executable, source, out, err)
+    try:
+        result = eval(out)
+    except Exception:
+        raise ExecFailed(executable, source, out,
+                         "could not decode %r" % out)
+    return result
+
+class ExecFailed(Exception):
+    def __init__(self, executable, source, out, err):
+        self.executable = executable
+        self.source = source
+        self.out = out
+        self.err = err
+
+class InterpreterInfo:
+    runnable = True
+
+    def __init__(self, name, executable, version_info):
+        assert name and executable and version_info
+        self.name = name
+        self.executable = executable
+        self.version_info = version_info
+
+    def __str__(self):
+        return "<executable at %s, version_info %s>" % (
+                self.executable, self.version_info)
+
+class NoInterpreterInfo:
+    runnable = False
+    def __init__(self, name, executable=None,
+                 out=None, err="not found"):
+        self.name = name
+        self.executable = executable
+        self.version_info = None
+        self.out = out
+        self.err = err
+
+    def __str__(self):
+        if self.executable:
+            return "<executable at %s, not runnable>"
+        else:
+            return "<executable not found for: %s>" % self.name
+
+if sys.platform != "win32":
+    def find_executable(name):
+        return py.path.local.sysfind(name)
+
+else:
+    # Exceptions to the usual windows mapping
+    win32map = {
+            'python': sys.executable,
+            'jython': "c:\jython2.5.1\jython.bat",
+    }
+    def locate_via_py(v_maj, v_min):
+        ver = "-%s.%s" % (v_maj, v_min)
+        script = "import sys; print(sys.executable)"
+        py_exe = py.path.local.sysfind('py')
+        if py_exe:
+            try:
+                exe = py_exe.sysexec(ver, '-c', script).strip()
+            except py.process.cmdexec.Error:
+                exe = None
+            if exe:
+                exe = py.path.local(exe)
+                if exe.check():
+                    return exe
+
+    def find_executable(name):
+        p = py.path.local.sysfind(name)
+        if p:
+            return p
+        actual = None
+        # Is this a standard PythonX.Y name?
+        m = re.match(r"python(\d)\.(\d)", name)
+        if m:
+            # The standard names are in predictable places.
+            actual = r"c:\python%s%s\python.exe" % m.groups()
+        if not actual:
+            actual = win32map.get(name, None)
+        if actual:
+            actual = py.path.local(actual)
+            if actual.check():
+                return actual
+        # The standard executables can be found as a last resort via the
+        # Python launcher py.exe
+        if m:
+            locate_via_py(*m.groups())
+
+def pyinfo():
+    import sys
+    return dict(version_info=tuple(sys.version_info))
+
+def sitepackagesdir(envdir):
+    from distutils.sysconfig import get_python_lib
+    return dict(dir=get_python_lib(envdir))
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.