Source

Templeton API / templeton.py

# -*- coding: utf-8 -*-
import urllib
import urllib2
import simplejson as json


TEMPLETON_URL = "http://ec2-75-101-199-141.compute-1.amazonaws.com:50111/templeton/v1/"


class Templeton(object):

    def __init__(self, user="hdfs"):
        self.user = user

    def get(self, url, data=None):
        """
        Make GET query to templeton url.
        """
        if data is not None:
            data['user.name'] = self.user
        else:
            data = {"user.name": self.user}
        data = urllib.urlencode(data)
        response = urllib2.urlopen(TEMPLETON_URL + url + "?" + data)
        return json.loads(response.read())

    def post(self, url, data=None):
        """
        Make POST query to templeton url.
        """
        if data is not None:
            data['user.name'] = self.user
        else:
            data = {"user.name": self.user}
        data = urllib.urlencode(data)
        req = urllib2.Request(TEMPLETON_URL + url, data)
        response = urllib2.urlopen(req)
        return json.loads(response.read())

    def delete(self, url, data=None):
        """
        Make DELETE query to templeton url.
        """
        if data is not None:
            data['user.name'] = self.user
        else:
            data = {"user.name": self.user}
        data = urllib.urlencode(data)
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        req = urllib2.Request(TEMPLETON_URL + url + "?" + data)
        req.get_method = lambda: 'DELETE'
        response = opener.open(req)
        return json.loads(response.read())


    def pig_query(self, execute=None, pig_file=None, statusdir=None, callback=None):
        """
        Create and queue a Pig job.

        Keyword arguments:
        user -- Hue/Hadoop user
        execute -- String containing an entire, short pig program to run. (e.g. pwd)
        file -- HDFS file name of a pig program to run. (One of either "execcute" or "file" is required )
        statusdir -- A directory where Templeton will write the status of the Pig job. If
                     provided, it is the caller's responsibility to remove this directory when done.
        callback -- Define a URL to Optional be called upon job completion. You may embed a specific job
                    ID into this URL using $jobId. This tag will be replaced in the callback URL with this job's job
                    ID.

        Returns dict:
        id -- A string containing the job ID similar to "job_201110132141_0001".
        info -- A JSON object containing the information returned when the job was queued.
        """
        if not any([execute, pig_file]):
            raise Exception("""One of either "execcute" or "file" is required""")
        data = {}
        if execute:
            data['execute'] = execute
        if pig_file:
            data['file'] = pig_file
        if statusdir:
            data['statusdir'] = statusdir
        if callback:
            data['callback'] = callback
        return self.post("pig", data)

    def check_job(self, job_id):
        """
        Check the status of a job and get related job information given its job ID.
        """
        return self.get("queue/%s" % job_id)

    def kill_job(self, job_id):
        return self.delete("queue/%s" % job_id)




if __name__ == "__main__":
    t = Templeton()
    job = t.pig_query(pig_file="/tmp/hcat.pig", statusdir="/tmp/hcat_output/")
    print "Started job %s " % job
    print "Job status %s " % t.check_job(job['id'])
    print "Killing job %s " % t.kill_job(job['id'])
    #print t.delete("ddl/database/default/table/a3")

    
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.