Source

cheesecake-service / cheesecake_daemon.py

Full commit
#!/usr/bin/env python2.4

import os
import re
import signal
import time

from config import CHEESECAKE_INDEX_PATH
from config import LOG_DIRECTORY
from config import MAX_CHEESECAKE_INDEX_TIMEOUT
from config import MAX_NUMBER_OF_FAILURES
from config import TIMESTAMP_FILE
from config import TIME_CORRECTION

from commands_stream import CommandsStream
from pypi import get_releases_list, post_results
from store import Store
from store import CheesecakeRun
from store import CheesecakeScore
from util import daemonize
from util import run_command
from util import touch_file
from util import fill_file_with


interrupt_signal_received = False

def set_interrupt_flag(signum, stack):
    """Signal handler.
    """
    interrupt_signal_received = True

default_sigint_signal = None

def signal_set():
    global default_sigint_signal
    default_sigint_signal = signal.signal(signal.SIGINT, set_interrupt_flag)

def signal_unset():
    signal.signal(signal.SIGINT, default_sigint_signal)

def score_package(number, name, version):
    """Score a package.

    Return (CheesecakeRun instance, CheesecakeScore instance) tuple.
    If run failed, score instance is set to None.
    """
    # Init CheesecakeRun instance.
    run = CheesecakeRun(number, name, version)
    logname = os.path.join(LOG_DIRECTORY, run.logname())

    # Run cheesecake_index with logfile saved into LOG_DIRECTORY.
    execution_start = time.time()
    code, output = run_command([CHEESECAKE_INDEX_PATH,
                                "--name",
                                "%s==%s" % (name, version),
                                "--pylint-max-execution-time=%d" % max(MAX_CHEESECAKE_INDEX_TIMEOUT, MAX_CHEESECAKE_INDEX_TIMEOUT - 60),
                                "--keep-log",
                                "--logfile=%s" % logname],
                               max_timeout=MAX_CHEESECAKE_INDEX_TIMEOUT)
    execution_end = time.time()

    # Update instance with execution time.
    run.execution_time = int(execution_end - execution_start)

    # Update instance with result.
    if code == 0 and not output.startswith('Error'):
        run.result = 'success'
        # Create CheesecakeScore instance from the script output.
        score = CheesecakeScore(output)
    else:
        run.result = 'failure'
        score = None

    new_logname = os.path.join(LOG_DIRECTORY, run.logname())

    # Rename logfile, so its name contains real result.
    if os.path.exists(logname):
        os.rename(logname, new_logname)
    else:
        # Create empty logfile and save Cheesecake output to it.
        fill_file_with(new_logname, output)

    return run, score

def name_filter(release):
    """Filter out all bad characters which setuptools don't like.

    >>> name_filter(("Endgame: Singularity", "0.25"))
    ('Endgame_Singularity', '0.25')
    >>> name_filter(('P(x)', '0.2'))
    ('P_x_', '0.2')
    >>> name_filter(('Ultimate WebShots Converter for KDE', '1.0'))
    ('Ultimate_WebShots_Converter_for_KDE', '1.0')
    """
    name, version = release
    name = re.sub(r'[\s:\(\)]+', '_', name)
    return (name, version)

def main():
    daemonize()

    # Create a command stream for communication with other processes.
    commands = CommandsStream()

    # Upon SIGINT daemon should gracefully terminate.
    signal_set()

    store = Store(directory=LOG_DIRECTORY, timestamp_file=TIMESTAMP_FILE)

    # Handy helper.
    def log_new_list_info(message, list):
        store.log(message % (len(list), ', '.join([('%s-%s' % tuple(x)) for x in list])))

    store.log("Cheesecake service started... How about a little red Leicester?")

    while not interrupt_signal_received:
        latest_scores = []

        # Don't get a new package list from PyPI until we score
        #     all packages from the previous list.
        while store.releases_to_score:
            new_releases_to_score = []

            for name, version in store.releases_to_score:
                run, score = score_package(store.runs_count, name, version)

                store.save_run(run)

                if run.result == 'success':
                    store.log("Package %s-%s scored successfully (installability: %d, documentation: %d, code kwalitee: %d) in %d seconds." % \
                              (name,
                               version,
                               score.installability.relative,
                               score.documentation.relative,
                               score.code_kwalitee.relative,
                               run.execution_time))
                    store.save_score(name, version, score)
                    latest_scores.append((name, version, score))

                if run.result == 'failure':
                    store.log("Failed to score package %s-%s in %d seconds. Details in log: %s." % \
                              (name,
                               version,
                               run.execution_time,
                               os.path.join(LOG_DIRECTORY, run.logname())))
                    if store.number_of_failures(name, version) < MAX_NUMBER_OF_FAILURES:
                        new_releases_to_score.append((name, version))

            store.releases_to_score = new_releases_to_score

        # Post results to PyPI.
        for name, version, score in latest_scores:
            if not post_results(name, version, score.serialize()):
                store.log("Cheesecake data rejected by PyPI for package %s-%s." % (name, version))
            # Do not contact PyPI to often.
            time.sleep(1)

        # Get new list of packages from PyPI.
        # Allow user to interrupt this one.
        signal_unset()
        try:
            to_score = []
            while not to_score:
                # Don't contact PyPI to often or RJ will get mad at us.
                if store.timestamp + 60 > int(time.time()):
                    time.sleep(60)

                # Add some seconds to overcome time differences between our host
                # and PyPI.
                to_score_from_pypi = get_releases_list(store.timestamp + TIME_CORRECTION)
                if to_score_from_pypi:
                    log_new_list_info("Got list of %d new packages from PyPI: %s.", to_score_from_pypi)

                to_score_from_stream = commands.get_releases_list()
                if to_score_from_stream:
                    log_new_list_info("Got list of %d new packages from command stream: %s.", to_score_from_stream)

                to_score = map(name_filter, to_score_from_pypi + to_score_from_stream)
        except Exception, e:
            if str(e):
                store.log("Interrupted by exception: %s. Terminating..." % str(e))
            else:
                store.log("Interrupted by user. Terminating...")
            break
        signal_set()

        store.releases_to_score = to_score
        store.update_timestamp()

    store.close()
    commands.close()


if __name__ == '__main__':
    main()