ash-mozharness / scripts /

#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at
# ***** END LICENSE BLOCK *****

import copy
import os
import re
import sys

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

from mozharness.base.errors import BaseErrorList
from mozharness.base.log import ERROR
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.testing.errors import LogcatErrorList
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser, EmulatorMixin
from mozharness.mozilla.tooltool import TooltoolMixin

class MarionetteUnittestOutputParser(DesktopUnittestOutputParser):
    A class that extends DesktopUnittestOutputParser such that it can
    catch failed gecko installation errors.

    bad_gecko_install = re.compile(r'Error installing gecko!')

    def __init__(self, **kwargs):
        self.install_gecko_failed = False
        super(MarionetteUnittestOutputParser, self).__init__(**kwargs)

    def parse_single_line(self, line):
            self.install_gecko_failed = True
        super(MarionetteUnittestOutputParser, self).parse_single_line(line)

class B2GEmulatorTest(TestingMixin, TooltoolMixin, EmulatorMixin, VCSMixin, BaseScript):
    test_suites = ('reftest', 'mochitest', 'xpcshell', 'crashtest')
    config_options = [
        {"action": "store",
         "dest": "test_type",
         "default": "browser",
         "help": "The type of tests to run",
        {"action": "store",
         "dest": "busybox_url",
         "default": None,
         "help": "URL to the busybox binary",
        {"action": "store",
         "dest": "emulator_url",
         "default": None,
         "help": "URL to the emulator zip",
        {"action": "store",
         "dest": "xre_url",
         "default": None,
         "help": "URL to the desktop xre zip",
        {"action": "store",
         "dest": "gecko_url",
         "default": None,
         "help": "URL to the gecko build injected into the emulator",
        {"action": "store",
         "dest": "test_manifest",
         "default": None,
         "help": "Path to test manifest to run",
        {"action": "store",
         "dest": "test_suite",
         "type": "choice",
         "choices": test_suites,
         "help": "Which test suite to run",
        {"action": "store",
         "dest": "adb_path",
         "default": None,
         "help": "Path to adb",
        {"action": "store",
         "dest": "total_chunks",
         "help": "Number of total chunks",
        {"action": "store",
         "dest": "this_chunk",
         "help": "Number of this chunk",
        }]] + copy.deepcopy(testing_config_options)

    error_list = [
        {'substr': 'FAILED (errors=', 'level': ERROR},
        {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
        {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
        {'regex': re.compile(r'''(Timeout|NoSuchAttribute|Javascript|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)Exception'''), 'level': ERROR},

    mozbase_dir = os.path.join('tests', 'mozbase')
    virtualenv_modules = [
        { 'manifestparser': os.path.join(mozbase_dir, 'manifestdestiny') },
        { 'mozfile': os.path.join(mozbase_dir, 'mozfile') },
        { 'mozhttpd': os.path.join(mozbase_dir, 'mozhttpd') },
        { 'mozinfo': os.path.join(mozbase_dir, 'mozinfo') },
        { 'mozinstall': os.path.join(mozbase_dir, 'mozinstall') },
        { 'mozprofile': os.path.join(mozbase_dir, 'mozprofile') },
        { 'mozprocess': os.path.join(mozbase_dir, 'mozprocess') },
        { 'mozrunner': os.path.join(mozbase_dir, 'mozrunner') },
        { 'mozdevice': os.path.join(mozbase_dir, 'mozdevice') },
        { 'marionette': os.path.join('tests', 'marionette') },

    def __init__(self, require_config_file=False):
        super(B2GEmulatorTest, self).__init__(
                'virtualenv_modules': self.virtualenv_modules,
                'require_test_zip': True,
                'emulator': 'arm',
                # This is a special IP that has meaning to the emulator
                'remote_webserver': '',

        # these are necessary since self.config is read only
        c = self.config
        self.adb_path = c.get('adb_path', self._query_adb())
        self.installer_url = c.get('installer_url')
        self.installer_path = c.get('installer_path')
        self.test_url = c.get('test_url')
        self.test_manifest = c.get('test_manifest')
        self.busybox_path = None

    # TODO detect required config items and fail if not set

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(B2GEmulatorTest, self).query_abs_dirs()
        dirs = {}
        dirs['abs_test_install_dir'] = os.path.join(
            abs_dirs['abs_work_dir'], 'tests')
        dirs['abs_xre_dir'] = os.path.join(
            abs_dirs['abs_work_dir'], 'xre')
        dirs['abs_emulator_dir'] = os.path.join(
            abs_dirs['abs_work_dir'], 'emulator')
        dirs['abs_b2g-distro_dir'] = os.path.join(
            dirs['abs_emulator_dir'], 'b2g-distro')
        dirs['abs_mochitest_dir'] = os.path.join(
            dirs['abs_test_install_dir'], 'mochitest')
        dirs['abs_modules_dir'] = os.path.join(
            dirs['abs_test_install_dir'], 'modules')
        dirs['abs_reftest_dir'] = os.path.join(
            dirs['abs_test_install_dir'], 'reftest')
        dirs['abs_crashtest_dir'] = os.path.join(
            dirs['abs_test_install_dir'], 'reftest')
        dirs['abs_xpcshell_dir'] = os.path.join(
            dirs['abs_test_install_dir'], 'xpcshell')
        for key in dirs.keys():
            if key not in abs_dirs:
                abs_dirs[key] = dirs[key]
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def download_and_extract(self):
        super(B2GEmulatorTest, self).download_and_extract()
        dirs = self.query_abs_dirs()
        if self.config.get('download_minidump_stackwalk'):


        if self.config['busybox_url']:
            self.busybox_path = os.path.join(dirs['abs_work_dir'], 'busybox')

    def _query_abs_base_cmd(self, suite):
        dirs = self.query_abs_dirs()
        cmd = [self.query_python_path('python')]

        str_format_values = {
            'adbpath': self.adb_path,
            'b2gpath': dirs['abs_b2g-distro_dir'],
            'emulator': self.config['emulator'],
            'logcat_dir': dirs['abs_work_dir'],
            'modules_dir': dirs['abs_modules_dir'],
            'remote_webserver': self.config['remote_webserver'],
            'xre_path': os.path.join(dirs['abs_xre_dir'], 'bin'),
            'test_manifest': self.test_manifest,
            'gecko_path': os.path.dirname(self.binary_path),
            'symbols_path': self.symbols_path,
            'busybox': self.busybox_path,
            'total_chunks': self.config.get('total_chunks'),
            'this_chunk': self.config.get('this_chunk'),

        name = '%s_options' % suite
        options = self.tree_config.get(name, self.config.get(name))
        if options:
            for option in options:
                option = option % str_format_values
                if not option.endswith('None'):
        return cmd

    def _query_adb(self):
        return self.which('adb') or \
               os.getenv('ADB_PATH') or \
                            'out', 'host', 'linux-x86', 'bin', 'adb')

    def _dump_logcat(self, parser):
        if parser.fail_count != 0 or parser.crashed or parser.leaked:
            dirs = self.query_abs_dirs()
            logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
            if os.access(logcat, os.F_OK):
      'dumping logcat')
                self.run_command(['cat', logcat], error_list=LogcatErrorList)
      'no logcat file found')

    def preflight_run_tests(self):
        super(B2GEmulatorTest, self).preflight_run_tests()
        suite = self.config['test_suite']
        # set default test manifest by suite if none specified
        if not self.test_manifest:
            if suite == 'mochitest':
                self.test_manifest = 'b2g.json'
            elif suite == 'reftest':
                self.test_manifest = os.path.join('tests', 'layout',
                                                  'reftests', 'reftest.list')
            elif suite == 'xpcshell':
                self.test_manifest = os.path.join('tests', 'xpcshell_b2g.ini')
            elif suite == 'crashtest':
                self.test_manifest = os.path.join('tests', 'testing',
                                                  'crashtest', 'crashtests.list')

        if not os.path.isfile(self.adb_path):
            self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)

    def run_tests(self):
        Run the tests
        dirs = self.query_abs_dirs()

        error_list = self.error_list

        suite = self.config['test_suite']
        if suite not in self.test_suites:
            self.fatal("Don't know how to run --test-suite '%s'!" % suite)

        cmd = self._query_abs_base_cmd(suite)
        cwd = dirs['abs_%s_dir' % suite]

        # TODO we probably have to move some of the code in
        # scripts/ and scripts/ to
        # mozharness.mozilla.testing.unittest so we can share it.
        # In the short term, I'm ok with some duplication of code if it
        # expedites things; please file bugs to merge if that happens.

        suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
        if self.config.get('this_chunk'):
            suite = '%s-%s' % (suite_name, self.config['this_chunk'])
            suite = suite_name

        # bug 773703
        success_codes = None
        if suite_name == 'xpcshell':
            success_codes = [0, 1]

        env = {}
        if self.query_minidump_stackwalk():
            env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
        env = self.query_env(partial_env=env)

        for i in range(0, 5):
            # We retry the run because sometimes installing gecko on the
            # emulator can cause B2G not to restart properly - Bug 812935.
            parser = MarionetteUnittestOutputParser(suite_category=suite_name,
            return_code = self.run_command(cmd, cwd=cwd, env=env,
            if not parser.install_gecko_failed:
            self.fatal("Failed to install gecko 5 times in a row, aborting")

        tbpl_status, log_level = parser.evaluate_parser(return_code)

        self.buildbot_status(tbpl_status, level=log_level)
        self.log("The %s suite: %s ran with return status: %s" %
                 (suite_name, suite, tbpl_status), level=log_level)

if __name__ == '__main__':
    emulatorTest = B2GEmulatorTest()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.