Commits

Chris Jerdonek committed 5630201

Address issue #125 by adding a --hashseed command-line option.

This commit also causes Tox to set PYTHONHASHSEED for test commands to a
random integer generated when tox is invoked. See the issue here:

https://bitbucket.org/hpk42/tox/issue/125

Comments (0)

Files changed (5)

doc/example/basic.txt

 from the ``subdir`` below the directory where your ``tox.ini``
 file resides.
 
+special handling of PYTHONHASHSEED
+-------------------------------------------
+
+.. versionadded:: 1.6.2
+
+By default, Tox sets PYTHONHASHSEED_ for test commands to a random integer
+generated when ``tox`` is invoked.  This mimics Python's hash randomization
+enabled by default starting `in Python 3.3`_.  To aid in reproducing test
+failures, Tox displays the value of ``PYTHONHASHSEED`` in the test output.
+
+You can tell Tox to use an explicit hash seed value via the ``--hashseed``
+command-line option to ``tox``.  You can also override the hash seed value
+per test environment in ``tox.ini`` as follows::
+
+    [testenv:hash]
+    setenv =
+        PYTHONHASHSEED = 100
+
+.. _`in Python 3.3`: http://docs.python.org/3/whatsnew/3.3.html#builtin-functions-and-types
+.. _PYTHONHASHSEED: http://docs.python.org/using/cmdline.html#envvar-PYTHONHASHSEED
+
 Integration with setuptools/distribute test commands
 ----------------------------------------------------
 

tests/test_config.py

 from textwrap import dedent
 
 import py
+import tox._config
 from tox._config import *
 from tox._config import _split_env
 
         assert envconfig.sitepackages == False
         assert envconfig.develop == False
         assert envconfig.envlogdir == envconfig.envdir.join("log")
-        assert envconfig.setenv is None
+        assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED']
+        hashseed = envconfig.setenv['PYTHONHASHSEED']
+        assert isinstance(hashseed, str)
+        # The following line checks that hashseed parses to an integer.
+        int_hashseed = int(hashseed)
+        # hashseed is random by default, so we can't assert a specific value.
+        assert int_hashseed > 0
 
     def test_installpkg_tops_develop(self, newconfig):
         config = newconfig(["--installpkg=abc"], """
         assert env.basepython == "python2.4"
         assert env.commands == [['xyz']]
 
+class TestHashseedOption:
+
+    def _get_envconfigs(self, newconfig, args=None, tox_ini=None,
+                        make_hashseed=None):
+        if args is None:
+            args = []
+        if tox_ini is None:
+            tox_ini = """
+                [testenv]
+            """
+        if make_hashseed is None:
+            make_hashseed = lambda: '123456789'
+        original_make_hashseed = tox._config.make_hashseed
+        tox._config.make_hashseed = make_hashseed
+        try:
+            config = newconfig(args, tox_ini)
+        finally:
+            tox._config.make_hashseed = original_make_hashseed
+        return config.envconfigs
+
+    def _get_envconfig(self, newconfig, args=None, tox_ini=None):
+        envconfigs = self._get_envconfigs(newconfig, args=args,
+                                          tox_ini=tox_ini)
+        return envconfigs["python"]
+
+    def _check_hashseed(self, envconfig, expected):
+        assert envconfig.setenv == {'PYTHONHASHSEED': expected}
+
+    def _check_testenv(self, newconfig, expected, args=None, tox_ini=None):
+        envconfig = self._get_envconfig(newconfig, args=args, tox_ini=tox_ini)
+        self._check_hashseed(envconfig, expected)
+
+    def test_default(self, tmpdir, newconfig):
+        self._check_testenv(newconfig, '123456789')
+
+    def test_passing_integer(self, tmpdir, newconfig):
+        args = ['--hashseed', '1']
+        self._check_testenv(newconfig, '1', args=args)
+
+    def test_passing_string(self, tmpdir, newconfig):
+        args = ['--hashseed', 'random']
+        self._check_testenv(newconfig, 'random', args=args)
+
+    def test_passing_empty_string(self, tmpdir, newconfig):
+        args = ['--hashseed', '']
+        self._check_testenv(newconfig, '', args=args)
+
+    def test_passing_no_argument(self, tmpdir, newconfig):
+        """Test that passing no arguments to --hashseed is not allowed."""
+        args = ['--hashseed']
+        try:
+            self._check_testenv(newconfig, '', args=args)
+        except SystemExit:
+            e = sys.exc_info()[1]
+            assert e.code == 2
+            return
+        assert False  # getting here means we failed the test.
+
+    def test_setenv(self, tmpdir, newconfig):
+        """Check that setenv takes precedence."""
+        tox_ini = """
+            [testenv]
+            setenv =
+                PYTHONHASHSEED = 2
+        """
+        self._check_testenv(newconfig, '2', tox_ini=tox_ini)
+        args = ['--hashseed', '1']
+        self._check_testenv(newconfig, '2', args=args, tox_ini=tox_ini)
+
+    def test_noset(self, tmpdir, newconfig):
+        args = ['--hashseed', 'noset']
+        envconfig = self._get_envconfig(newconfig, args=args)
+        assert envconfig.setenv is None
+
+    def test_noset_with_setenv(self, tmpdir, newconfig):
+        tox_ini = """
+            [testenv]
+            setenv =
+                PYTHONHASHSEED = 2
+        """
+        args = ['--hashseed', 'noset']
+        self._check_testenv(newconfig, '2', args=args, tox_ini=tox_ini)
+
+    def test_one_random_hashseed(self, tmpdir, newconfig):
+        """Check that different testenvs use the same random seed."""
+        tox_ini = """
+            [testenv:hash1]
+            [testenv:hash2]
+        """
+        next_seed = [1000]
+        # This function is guaranteed to generate a different value each time.
+        def make_hashseed():
+            next_seed[0] += 1
+            return str(next_seed[0])
+        # Check that make_hashseed() works.
+        assert make_hashseed() == '1001'
+        envconfigs = self._get_envconfigs(newconfig, tox_ini=tox_ini,
+                                          make_hashseed=make_hashseed)
+        self._check_hashseed(envconfigs["hash1"], '1002')
+        # Check that hash2's value is not '1003', for example.
+        self._check_hashseed(envconfigs["hash2"], '1002')
+
+    def test_setenv_in_one_testenv(self, tmpdir, newconfig):
+        """Check using setenv in one of multiple testenvs."""
+        tox_ini = """
+            [testenv:hash1]
+            setenv =
+                PYTHONHASHSEED = 2
+            [testenv:hash2]
+        """
+        envconfigs = self._get_envconfigs(newconfig, tox_ini=tox_ini)
+        self._check_hashseed(envconfigs["hash1"], '2')
+        self._check_hashseed(envconfigs["hash2"], '123456789')
+
 class TestIndexServer:
     def test_indexserver(self, tmpdir, newconfig):
         config = newconfig("""

tests/test_venv.py

 import tox
 import pytest
 import os, sys
+import tox._config
 from tox._venv import *
 
 py25calls = int(sys.version_info[:2] == (2,5))
     venv.update()
     mocksession.report.expect("verbosity0", "*recreate*")
 
+def test_test_hashseed_is_in_output(newmocksession):
+    original_make_hashseed = tox._config.make_hashseed
+    tox._config.make_hashseed = lambda: '123456789'
+    try:
+        mocksession = newmocksession([], '''
+            [testenv]
+        ''')
+    finally:
+        tox._config.make_hashseed = original_make_hashseed
+    venv = mocksession.getenv('python')
+    venv.update()
+    venv.test()
+    mocksession.report.expect("verbosity0", "python runtests: PYTHONHASHSEED='123456789'")
+
 def test_test_runtests_action_command_is_in_output(newmocksession):
     mocksession = newmocksession([], '''
         [testenv]
 import argparse
 import distutils.sysconfig
 import os
+import random
 import sys
 import re
 import shlex
              "all commands and results involved.  This will turn off "
              "pass-through output from running test commands which is "
              "instead captured into the json result file.")
+    # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED.
+    parser.add_argument("--hashseed", action="store",
+        metavar="SEED", default=None,
+        help="set PYTHONHASHSEED to SEED before running commands.  "
+             "Defaults to a random integer in the range 1 to 4294967295.  "
+             "Passing 'noset' suppresses this behavior.")
     parser.add_argument("args", nargs="*",
         help="additional arguments available to command positional substitution")
     return parser
     except Exception:
         return None
 
+def make_hashseed():
+    return str(random.randint(1, 4294967295))
+
 class parseini:
     def __init__(self, config, inipath):
         config.toxinipath = inipath
         else:
             raise ValueError("invalid context")
 
+        if config.option.hashseed is None:
+            hashseed = make_hashseed()
+        elif config.option.hashseed == 'noset':
+            hashseed = None
+        else:
+            hashseed = config.option.hashseed
+        config.hashseed = hashseed
 
         reader.addsubstitutions(toxinidir=config.toxinidir,
                                 homedir=config.homedir)
                         arg = vc.changedir.bestrelpath(origpath)
                     args.append(arg)
             reader.addsubstitutions(args)
-        vc.setenv = reader.getdict(section, 'setenv')
+        setenv = {}
+        if config.hashseed is not None:
+            setenv['PYTHONHASHSEED'] = config.hashseed
+        setenv.update(reader.getdict(section, 'setenv'))
+        vc.setenv = setenv
         if not vc.setenv:
             vc.setenv = None
 
             self.run_install_command(packages=packages, options=options,
                                      action=action, extraenv=extraenv)
 
-    def _getenv(self):
-        env = self.envconfig.setenv
-        if env:
-            env_arg = os.environ.copy()
-            env_arg.update(env)
-        else:
-            env_arg = None
-        return env_arg
+    def _getenv(self, extraenv={}):
+        env = os.environ.copy()
+        setenv = self.envconfig.setenv
+        if setenv:
+            env.update(setenv)
+        env.update(extraenv)
+        return env
 
     def test(self, redirect=False):
         action = self.session.newaction(self, "runtests")
             self.status = 0
             self.session.make_emptydir(self.envconfig.envtmpdir)
             cwd = self.envconfig.changedir
+            env = self._getenv()
+            # Display PYTHONHASHSEED to assist with reproducibility.
+            action.setactivity("runtests", "PYTHONHASHSEED=%r" % env.get('PYTHONHASHSEED'))
             for i, argv in enumerate(self.envconfig.commands):
                 # have to make strings as _pcall changes argv[0] to a local()
                 # happens if the same environment is invoked twice
         old = self.patchPATH()
         try:
             args[0] = self.getcommandpath(args[0], venv, cwd)
-            env = self._getenv() or os.environ.copy()
-            env.update(extraenv)
+            env = self._getenv(extraenv)
             return action.popen(args, cwd=cwd, env=env, redirect=redirect)
         finally:
             os.environ['PATH'] = old