Source

pygamebuilder / update_build.py

import os
import sys
import re
import time
import shutil
import ConfigParser
from collections import defaultdict
from glob import glob
from subprocess import check_output

import callproc

REPO_URL = 'https://bitbucket.org/pygame/pygame'
REPO_DIR = os.path.join('source', 'pygame')

def hg(*args, **kwargs):
    "Simple wrapper for calling hg command"
    return check_output(('hg',) + args, cwd=kwargs.get('cwd', REPO_DIR))

def update_version_file(revhash):
    """Update lib/version.py with the hash from hg."""
    target = os.path.join(REPO_DIR, 'lib', 'version.py')
    with open(target, 'r') as f:
        contents = f.read()
    contents, nsub = re.subn (r"(ver\s*=\s*)'([0-9]+\.[0-9]+\.[0-9]+[^']*)'", r"\1'\2-hg-%s'" % revhash, contents)
    assert nsub > 0, "Didn't find version number in %s" % target
    with open(target, 'w') as f:
        f.write(contents)

def write_file_lines(filename, line_list):
    with open(filename, "w") as file_obj:
        for line in line_list:
            if not isinstance(line, str):
                line = str(line)
            file_obj.write(line)
            file_obj.write("\n")

blame_line_re = re.compile('\s*([\w\-]+) (\w+): (.*)')
def AppendBlameInfoToErrorsByFile(errors_by_file):
    """Get blame from hg, and add it to error info.
    
    Adds 3 fields to each error: user, revision hash & line content.
    """
    for error_file in errors_by_file:
        print "blame for",error_file
        try:
            blame_output = hg('blame', '-u', '-c', error_file)
        except:
            # The tests produce incorrect filenames sometimes
            error.extend(('',)*3)
            continue
        
        blame_lines = blame_output.splitlines()
        for error in errors_by_file[error_file]:
            line = int(error[0])
            line_match = blame_line_re.match(blame_lines[line - 1])
            if line_match is None:
                print line, repr(blame_lines[line - 1])
            error.extend(line_match.groups())

def GetBuildWarningsHTML(REPO_DIR, build_output):
    warnings_by_file = defaultdict(list)
    warning_matches = re.findall(r"^([^\(\s]+\.c)(?:\(|:)([0-9]+)(?:\)|:) ?:? warning:? ([^\r\n]+)[\r\n]", build_output, re.MULTILINE)
    if len(warning_matches) > 0:
        print "WARNING - found",len(warning_matches),"warnings"
        for warning_match in warning_matches:
            warning_file, line, message = warning_match
            warnings_by_file[warning_file].append([line, message])

        AppendBlameInfoToErrorsByFile(warnings_by_file)
                
        web_friendly_warnings = []
        for warning_file in warnings_by_file:
            for lineno, msg, user, line, rev in warnings_by_file[warning_file]:
                file_location = os.path.basename(warning_file) + ":" + lineno + " last rev: " + rev + ":" + user
                code_line = line.replace("<", "&lt;").replace(">", "&gt;").replace(" ", "&nbsp;")
                web_friendly_warnings.append(file_location + "<br>warning:" + msg + '<br><code>' + code_line + '</code>')                
        
        return "<hr>".join(web_friendly_warnings)

def run_tests(config_file, revno, revhash):
    """Run the tests for the latest version on a particular Python installation.
    """
    config_data = ConfigParser.SafeConfigParser()
    config_data.read([config_file])
    platform_id = os.path.basename(config_file).replace(".ini", "").replace("build_", "")

    last_rev_filename = "./output/last_rev_"+platform_id+".txt"

    assert(config_data.has_option("DEFAULT", "python_path"))
    python_path = config_data.get("DEFAULT", "python_path")
    assert os.path.exists(python_path), "Python not found at %s" % python_path

    print "-------------------------"
    print "building", platform_id, "with python at", python_path
    try:
        with open(last_rev_filename, 'r') as f:
            previous_rev = f.read().strip()
    except IOError:
        print "WARNING: could not find last rev built"
        previous_rev = ''
    
    if previous_rev == revhash:
        print "exiting - already built rev", revhash
        return
    
    print "building %s:%s (last built %s)" % (revno, revhash[:7], previous_rev)
    valid_build_attempt = True
    
    build_env = os.environ.copy()
    for option in config_data.options("build_env"):
        build_env[option] = config_data.get("build_env", option)
        
    ret_code, output = callproc.InteractiveGetReturnCodeAndOutput([python_path, "config.py"], "Y\nY\nY\n", REPO_DIR, build_env)
    print output
    if ret_code != 0:
        print "ERROR running config.py!"
        assert(ret_code == 0)

    dist_path = os.path.join(REPO_DIR, "dist")
    if os.path.exists(dist_path):
        shutil.rmtree(dist_path)

    package_command = config_data.get("DEFAULT", "make_package")
    ret_code, build_output = callproc.GetReturnCodeAndOutput([python_path, "setup.py", package_command], REPO_DIR, build_env)
    build_warnings = GetBuildWarningsHTML(REPO_DIR, output)
    if ret_code == 0:
        
        package_mask = config_data.get("DEFAULT", "package_mask")
        installer_glob = os.path.join(dist_path, package_mask)
        print "Looking for installer at:", installer_glob
        installer_dist_path = glob(os.path.join(dist_path, package_mask))[0]
        print "got installer at:", installer_dist_path
        installer_filename = os.path.split(installer_dist_path)[1]
        installer_path = "./output/"+installer_filename
        shutil.move(installer_dist_path, installer_path)
        
        temp_install_path = os.path.join(os.getcwd(), "install_test")
        if os.path.exists(temp_install_path):
            shutil.rmtree(temp_install_path)
        os.mkdir(temp_install_path)

        test_subpath = config_data.get("DEFAULT", "test_dir_subpath")
        temp_install_pythonpath = os.path.join(temp_install_path, test_subpath)
        os.makedirs(temp_install_pythonpath)
        
        test_env = {"PYTHONPATH":temp_install_pythonpath}
        install_env = build_env.copy()
        install_env.update(test_env)

        print "installing to:",temp_install_path
        callproc.ExecuteAssertSuccess([python_path, "setup.py", "install", "--prefix", temp_install_path], REPO_DIR, install_env)
    
        print "running tests..."
        ret_code, output = callproc.GetReturnCodeAndOutput([python_path, "run_tests.py"], REPO_DIR, test_env)
        error_match = re.search("FAILED \([^\)]+=([0-9]+)\)", output)
        if ret_code != 0 or error_match != None:
            errors_by_file = defaultdict(list)
            error_matches = error_matches = re.findall(r"^((?:ERROR|FAIL): [^\n]+)\n+-+\n+((?:[^\n]+\n)+)\n", output, re.MULTILINE)
            if len(error_matches) > 0:
                print "TESTS FAILED - found",len(error_matches),"errors"
                for error_match in error_matches:
                    message, traceback = error_match
                    trace_top_match = re.search(r'File "([^"]+)", line ([0-9]+)', traceback)
                    error_file, line = trace_top_match.groups()
                    #~ print error_file, line, traceback
                    #~ return
                    errors_by_file[error_file].append([line, message, traceback])
                AppendBlameInfoToErrorsByFile(errors_by_file)
                
                for error_file in errors_by_file:
                    print "test failures in:", error_file
                    for error in errors_by_file[error_file]:
                        print error
                        
                build_result = "Build Successful, Tests FAILED"                            
                web_friendly_errors = []
                for error_file in errors_by_file:
                    for error in errors_by_file[error_file]:
                        file_location = os.path.split(error_file)[1] + ":" + error[0] + " last rev: " + error[-1] + ":" + error[-3] 
                        web_friendly_errors.append(file_location + "<br>" + error[1])                
                build_errors = "<hr>".join(web_friendly_errors)
            else:
                build_result = "Build Successful, Invalid Test Results"
                build_errors = output.replace("\n", "<br>")                            
                print "ERROR - tests failed! could not parse output:"
                print output
        else:   
            print "success! uploading..."
            result_filename = "./output/prebuilt_%s.txt" % platform_id
            write_file_lines(result_filename, [revhash, time.strftime("%Y-%m-%d %H:%M"), "uploading"])
            upload_results.scp(result_filename)
            upload_results.scp(installer_path)
            write_file_lines(result_filename, [revhash, time.strftime("%Y-%m-%d %H:%M"), installer_filename])
            upload_results.scp(result_filename)
            build_result = "Build Successful, Tests Passed"                            
            tests_run = re.findall(r"^loading ([\r\n]+)$", output, re.MULTILINE)
            test_text = [test + " passed" for test in tests_run]
            build_errors = "<br>".join(test_text)
    else:
        # Build failure
        error_matches = re.findall(r"^([^\(\s]+\.c)(?:\(|:)([0-9]+)(?:\)|:) ?:? error:? ([^\r\n]+)[\r\n]", build_output, re.MULTILINE)
        if len(error_matches) > 0:
            print "FAILED - found", len(error_matches), "errors"
            errors_by_file = {}
            for error_match in error_matches:
                error_file, line, message = error_match
                if error_file not in errors_by_file:
                    errors_by_file[error_file] = []
                errors_by_file[error_file].append([line, message])

            AppendBlameInfoToErrorsByFile(errors_by_file)
                    
            for error_file in errors_by_file:
                print "errors in:",error_file
                for error in errors_by_file[error_file]:
                    print error

            build_result = "Build FAILED, Tests not run"                            
            web_friendly_errors = []
            for error_file in errors_by_file:
                for error in errors_by_file[error_file]:
                    file_location = os.path.split(error_file)[1] + ":" + error[0] + " last rev: " + error[-1] + ":" + error[-3] 
                    web_friendly_errors.append(file_location + "<br>ERROR:" + error[1])                
            build_errors = "<hr>".join(web_friendly_errors)
        else:
            # Build failure, didn't find compile error messages.
            link_error_matches = re.findall(r"^([^\(\s]+)\.obj : error ([^\r\n]+)[\r\n]", build_output, re.MULTILINE)
            if len(link_error_matches) > 0:
                build_result = "Link FAILED, Tests not run"                           
                print "FAILED - found",len(link_error_matches),"errors"
                build_errors = ""
                for error_match in link_error_matches:
                    source_name, message = error_match
                    build_errors += source_name + " : " + message + "<br>"
                
            else:
                # Build failure, didn't find compile or link error messages.
                exception_match = re.search(r"^Traceback \(most recent call [a-z]+\):[\r\n]+(.+[^\r\n]+Error:[^\r\n]+)", build_output, re.MULTILINE | re.DOTALL)
                if exception_match != None:
                    build_result = "Build FAILED, Tests not run"                            
                    build_errors = exception_match.group(1).replace("\n", "<br>")
                        
                else:
                    # Build failure, didn't find compile, link or Python errors
                    build_result = "Build FAILED, Tests not run"                           
                    build_errors = ""
                    error_matches = re.findall(r"^error: ([^\r\n]+)", output, re.MULTILINE)
                    for error_match in error_matches:
                        build_errors += error_match + "<br>"

                    print "FAILED - unrecognized errors in:"
                    print build_output
    
    if valid_build_attempt:            
        result_filename = "./output/buildresults_%s.txt" % platform_id
        write_file_lines(result_filename, [revhash, time.strftime("%Y-%m-%d %H:%M"), build_result, build_errors, build_warnings])
        #upload_results.scp(result_filename)
        with open(last_rev_filename, 'w') as f:
            f.write(revhash)
        print "COMPLETED build of", revhash
        print "-------------------------"
    else:
        print "FAILED build attempt of", revhash


def main():
    """Update the repository & run the tests with the specified configurations."""
    script_path = os.path.dirname(sys.argv[0])
    print 'executing pygamebuilder from:', script_path
    if script_path != "": 
        os.chdir(script_path)
    print "-------------------------"

    # Set up required directories.
    if not os.path.exists("source"):
        os.mkdir("source")
    if not os.path.exists("output"):
        os.mkdir("output")
    
    # Ensure we've got the repository
    if not os.path.isdir(REPO_DIR):
        hg('clone', REPO_URL, REPO_DIR, cwd=None)
    
    # Ensure the repository is up to date
    #hg('pull', REPO_URL)
    hg('update', '-C', 'default')
    
    revno, revhash = hg('parents', '--template', '{rev} {node}').decode('ascii').split()
    update_version_file(revhash)
    
    if len(sys.argv) > 1:
        config_file_list = [os.path.join('config', 'build_%s.ini' % a) for a in sys.argv[1:]]
    else:
        config_file_list = glob(os.path.join('config', 'build_*.ini'))
    
    for config_file in config_file_list:
        run_tests(config_file, revno, revhash)

if __name__ == '__main__':
    main()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.