Peter Rudenko avatar Peter Rudenko committed e7e70a6

Templeton pig API

Comments (0)

Files changed (1)

+# -*- coding: utf-8 -*-
+import urllib
+import urllib2
+import simplejson as json
+
+
+TEMPLETON_URL = "http://ec2-75-101-199-141.compute-1.amazonaws.com:50111/templeton/v1/"
+
+
+class Templeton(object):
+
+    def __init__(self, user="hdfs"):
+        self.user = user
+
+    def get(self, url, data=None):
+        """
+        Make GET query to templeton url.
+        """
+        if data is not None:
+            data['user.name'] = self.user
+        else:
+            data = {"user.name": self.user}
+        data = urllib.urlencode(data)
+        response = urllib2.urlopen(TEMPLETON_URL + url + "?" + data)
+        return json.loads(response.read())
+
+    def post(self, url, data=None):
+        """
+        Make POST query to templeton url.
+        """
+        if data is not None:
+            data['user.name'] = self.user
+        else:
+            data = {"user.name": self.user}
+        data = urllib.urlencode(data)
+        req = urllib2.Request(TEMPLETON_URL + url, data)
+        response = urllib2.urlopen(req)
+        return json.loads(response.read())
+
+
+    def pig_query(self, execute=None, pig_file=None, statusdir=None, callback=None):
+        """
+        Create and queue a Pig job.
+
+        Keyword arguments:
+        user -- Hue/Hadoop user
+        execute -- String containing an entire, short pig program to run. (e.g. pwd)
+        file -- HDFS file name of a pig program to run. (One of either "execcute" or "file" is required )
+        statusdir -- A directory where Templeton will write the status of the Pig job. If
+                     provided, it is the caller's responsibility to remove this directory when done.
+        callback -- Define a URL to Optional be called upon job completion. You may embed a specific job
+                    ID into this URL using $jobId. This tag will be replaced in the callback URL with this job's job
+                    ID.
+
+        Returns dict:
+        id -- A string containing the job ID similar to "job_201110132141_0001".
+        info -- A JSON object containing the information returned when the job was queued.
+        """
+        if not any([execute, pig_file]):
+            raise Exception("""One of either "execcute" or "file" is required""")
+        data = {}
+        if execute:
+            data['execute'] = execute
+        if pig_file:
+            data['file'] = pig_file
+        if statusdir:
+            data['statusdir'] = statusdir
+        if callback:
+            data['callback'] = callback
+        return self.post("pig", data)
+
+    def check_job(self, job_id):
+        """
+        Check the status of a job and get related job information given its job ID.
+        """
+        return self.get("queue/%s" % job_id)
+
+
+
+
+if __name__ == "__main__":
+    t = Templeton()
+    pig_query = t.pig_query(user="hdfs", )
+    job = t.check_job(pig_query['id'])
+    print job
+    
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.