pythongrid / example.py

#!/usr/bin/env python

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# Written (W) 2008-2009 Christian Widmer
# Copyright (C) 2008-2009 Max-Planck-Society

import sys
import getopt
import pythongrid
from pythongrid import Job, Usage
from pythongrid import process_jobs, submit_jobs, collect_jobs, get_status
import time

#needs to be imported, such that module name can be referred to explicitly
import example


def compute_factorial(n):
    """
    computes factorial of n
    """
    ret=1
    for i in range(n):
        ret=ret*(i+1)

    return ret

def make_jobs():
    """
    Creates a list of Jobs.
    """
    inputvec = [[3], [5], [10], [15]]
    print 'print computing the factorials of %s' % str(inputvec)
    jobs = []

    for input in inputvec:
        # We need to use the full identifier
        # such that the module name is explicit.
        job = Job(example.compute_factorial, input, cleanup=True) 
        jobs.append(job)

    return jobs

def local_multithreading():
    """Run jobs on local machine"""
    print "===  Local Multithreading  ==="
    jobs = make_jobs()

    print "output ret field in each job before multithreaded computation"
    for (i, job) in enumerate(jobs):
        print "Job #", i, "- ret: ", job.ret

    print "executing jobs on local machine using 3 threads"
    if not pythongrid.multiprocessing_present:
        print 'multiprocessing not found, serial computation'

    processed_jobs = process_jobs(jobs, local=True, max_num_threads=3)

    print "ret fields AFTER execution on local machine"
    for (i, job) in enumerate(processed_jobs):
        print "Job #", i, "- ret: ", str(job.ret)[0:10]
    
def submit_and_wait():
    """Submit jobs to the cluster and wait for jobs to complete."""
    print "===  Submit and Wait  ==="
    jobs = make_jobs()

    print "output ret field in each job before sending it onto the cluster"
    for (i, job) in enumerate(jobs):
        print "Job #", i, "- ret: ", job.ret

    print "sending function jobs to cluster"
    processed_jobs = process_jobs(jobs)

    print "ret fields AFTER execution on cluster"
    for (i, job) in enumerate(processed_jobs):
        print "Job #", i, "- ret: ", str(job.ret)[0:10]

def submit_and_forget():
    """Submit jobs to cluster, and check them later."""
    print "===  Submit and Forget  ==="
    myjobs = make_jobs()

    (sid, jobids) = submit_jobs(myjobs)

    print 'checking whether finished'
    while not get_status(sid, jobids):
        time.sleep(7)
    print 'collecting jobs'
    retjobs = collect_jobs(sid, jobids, myjobs)
    print "ret fields AFTER execution on cluster"
    for (i, job) in enumerate(retjobs):
        print "Job #", i, "- ret: ", str(job.ret)[0:10]

def run_example():
    """
    execute example    
    """ 
    local_multithreading()
    submit_and_wait()
    submit_and_forget()


def main(argv=None):
    if argv is None:
        argv = sys.argv

    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
            run_example()

        except getopt.error, msg:
            raise Usage(msg)

    except Usage, err:
        print >> sys.stderr, err.msg
        print >> sys.stderr, "for help use --help"

        return 2


if __name__ == "__main__":
    main()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.