Commits

Anonymous committed ec3176e

Updated to python drmaa-0.4b1-r54

Comments (0)

Files changed (5)

     if not pythongrid.multiprocessing_present:
         print 'multiprocessing not found, serial computation'
 
-    processed_jobs = process_jobs(jobs, local=True, maxNumThreads=3)
+    processed_jobs = process_jobs(jobs, local=True, max_num_threads=3)
 
     print "ret fields AFTER execution on local machine"
     for (i, job) in enumerate(processed_jobs):
 # location of pythongrid.py on cluster file system
 # TODO set this in configuration file
 # PYGRID = "/absolute/path/to/pythongrid/pythongrid.py"
-#PYGRID = "/cluster/home/infk/cong/pythongrid/pythongrid.py"
+PYGRID = "/cluster/home/infk/cong/pythongrid/pythongrid.py"
 if PYGRID == "/absolute/path/to/pythongrid/pythongrid.py":
     print "Warning: variable PYGRID seems not to be set"
 
 # (must be writable from cluster)
 # TODO define separate client/server TEMPDIR
 # TEMPDIR = "/absolute/path/to/inputoutputdir/"
-#TEMPDIR = "/cluster/home/infk/cong/tmp/"
+TEMPDIR = "/cluster/home/infk/cong/tmp/"
 if TEMPDIR == "/absolute/path/to/inputoutputdir/":
     print "Warning: variable TEMPDIR seems not to be set"
     
         Input data is removed after execution to save space.
         """
         try:
-            self.ret = apply(self.func, self.args, self.kwlist)
+            #self.ret = apply(self.func, self.args, self.kwlist)
+            if self.kwlist:
+                self.ret = self.func(*(self.args), **(self.kwlist))
+            else:
+                self.ret = self.func(*(self.args))
 
         except Exception, error:
 
     """Cannot pickle method instances, so fake a function.
     Used by _process_jobs_locally"""
     
-    return apply(job.f, job.args, job.kwlist)
-
+    #return apply(job.f, job.args, job.kwlist)
+    if job.kwlist:
+        return  job.f(*(job.args), **(job.kwlist))
+    else:
+        return  job.f(*(job.args))
 
 
 def _process_jobs_locally(jobs, max_num_threads=None):
     """
 
     s = drmaa.Session()
+    s.initialize()
+    #sid = s.contact
+    #print "Started drmaa session with ID: %s" % sid
+    
     jobids = []
 
     for job in jobs:
 
         # set job fields that depend on the jobid assigned by grid engine
         job.jobid = jobid
-        log_stdout_fn = (os.path.expanduser(TEMPDIR) + job.name + '.o' + jobid)
-        log_stderr_fn = (os.path.expanduser(TEMPDIR) + job.name + '.e' + jobid)
+        #log_stdout_fn = (os.path.expanduser(TEMPDIR) + job.name + '.o' + jobid)
+        #log_stderr_fn = (os.path.expanduser(TEMPDIR) + job.name + '.e' + jobid)
+        log_output = (os.getcwd() + '/lsf.o' + jobid)
 
         print 'Your job %s has been submitted with id %s' % (job.name, jobid)
-        print "stdout:", log_stdout_fn
-        print "stderr:", log_stderr_fn
+        print "logfile:", log_output
         print ""
 
         #display tmp file size
         jobids.append(jobid)
         s.deleteJobTemplate(jt)
 
-    #sid = s.getContact()
+    #s.exit()
 
     return (s, jobids)
 
     #attempt to collect results
     ret_jobs = []
     for ix, job in enumerate(joblist):
-        log_stdout_fn = (os.path.expanduser(TEMPDIR) + job.name + '.o' + jobids[ix])
-        log_stderr_fn = (os.path.expanduser(TEMPDIR) + job.name + '.e' + jobids[ix])
+        #log_stdout_fn = (os.path.expanduser(TEMPDIR) + job.name + '.o' + jobids[ix])
+        #log_stderr_fn = (os.path.expanduser(TEMPDIR) + job.name + '.e' + jobids[ix])
+        log_output = (os.getcwd() + '/lsf.o' + jobids[ix])
         
         try:
             ret_job = load(job.outputfile)
             if ret_job.exception != None:
                 print str(type(ret_job.exception))
                 print "Exception encountered in job with log file:"
-                print log_stdout_fn
+                print log_output
                 print ret_job.exception
 
             #remove files
             elif ret_job.cleanup:
-
-                print "cleaning up:", job.outputfile
+                #print "cleaning up:", job.outputfile
                 os.remove(job.outputfile)
 
                 if ret_job != None:
-
-                    print "cleaning up:", log_stdout_fn
-                    os.remove(log_stdout_fn)
-
-                    #print "cleaning up:", log_stderr_fn
-                    #os.remove(log_stderr_fn)
-
+                    #print "cleaning up:", log_output
+                    os.remove(log_output)
 
         except Exception, error:
             print "error while unpickling file: " + job.outputfile
             print "this could caused by a problem with"
             print "the cluster environment, imports or environment variables"
             print "check log files for more information: "
-            print "stdout:", log_stdout_fn
-            print "stderr:", log_stderr_fn
+            print "stdout:", log_output
             
             print error
 
-
-
     return ret_jobs
 
 
     #s = drmaa.Session()
     #s.initialize(sid)
     s = sid
+
     status_summary = {}.fromkeys(_decodestatus, 0)
     for jobid in jobids:
         try:
 Your job has been submitted with id xxxxxx
 DRMAA quit successfully
 
+test_drmaa_twice
+----------------
+There is a bug in LSF which causes a segfault.
+
+gcc -o test_drmaa_twice -ldrmaa test_drmaa_twice.c
+./test_drmaa_twice
+
+The output is:
+DRMAA library was started successfully
+DRMAA job.
+Job <xxxxxx> is submitted to queue <xxxx>.
+Your job has been submitted with id xxxxxx
+starting second job
+DRMAA job.
+Job <xxxxxx> is submitted to queue <xxxx>.
+Your job has been submitted with id xxxxxx
+DRMAA quit successfully
+
 
 test_drmaa.py
 -------------
 DRM system used: XXXX
 DRMAA job.
 Job <xxxxxx> is submitted to queue <xxxx>.
+DRMAA job.
+Job <xxxxxx> is submitted to queue <xxxx>.
+Job exited using the following resources:
+{'mem': 'xx', 'start_time': 'xx', 'n_hosts': 'xx', 'vmem': 'xx'}
 Job exited using the following resources:
 {'mem': 'xx', 'start_time': 'xx', 'n_hosts': 'xx', 'vmem': 'xx'}
 
 from drmaa import *
 
 s = Session()
+s.initialize()
 print('DRMAA library v%s'%str(s.version))
 print('DRM system used: '+s.drmsInfo)
 
 jt.remoteCommand = '/bin/sleep'
 jt.args = ['10']
 jt.nativeSpecification = ''
-jname = s.runJob(jt)
-
-jinfo = s.wait(jname, Session.TIMEOUT_WAIT_FOREVER)
+jname1 = s.runJob(jt)
 
 jt2 = JobTemplate()
 jt2.remoteCommand = '/bin/sleep'
 jt2.args = ['20']
 jt2.nativeSpecification = ''
-jname = s.runJob(jt2)
+jname2 = s.runJob(jt2)
 
-jinfo2 = s.wait(jname, Session.TIMEOUT_WAIT_FOREVER)
+jinfo = s.wait(jname1, Session.TIMEOUT_WAIT_FOREVER)
+jinfo2 = s.wait(jname2, Session.TIMEOUT_WAIT_FOREVER)
 
 # Show all the returned information
 # print jinfo
     print jinfo.resourceUsage
 if (jinfo.hasSignal):
     print('Job was signalled with '+jinfo.terminatedSignal)
+if (jinfo2.wasAborted):
+    print('Job never ran')
+if (jinfo2.hasExited):
+    print('Job exited using the following resources:')
+    print jinfo.resourceUsage
+if (jinfo2.hasSignal):
+    print('Job was signalled with '+jinfo2.terminatedSignal)
 s.exit()
                          
 

test/test_drmaa_wait.c

-/* Start the drmaa library, submit a job (sleep 10), and quit. */
-
-#include <drmaa.h>
-
-int main (int argc, char **argv) {
-  char error[DRMAA_ERROR_STRING_BUFFER];
-  int errnum = 0;
-  drmaa_job_template_t *jt = NULL;
-
-  errnum = drmaa_init (NULL, error, DRMAA_ERROR_STRING_BUFFER);
-  
-  if (errnum != DRMAA_ERRNO_SUCCESS) {
-    fprintf (stderr, "Could not initialize the DRMAA library: %s\n", error);
-    return 1;
-  }
-  printf ("DRMAA library was started successfully\n");
-  
-  errnum = drmaa_allocate_job_template (&jt, error, DRMAA_ERROR_STRING_BUFFER);
-  
-  if (errnum != DRMAA_ERRNO_SUCCESS) {
-    fprintf (stderr, "Could not create job template: %s\n", error);
-  }
-  else {
-    errnum = drmaa_set_attribute (jt, DRMAA_REMOTE_COMMAND, "/bin/sleep", \
-				  error, DRMAA_ERROR_STRING_BUFFER);
-    
-    if (errnum != DRMAA_ERRNO_SUCCESS) {
-      fprintf (stderr, "Could not set attribute \"%s\": %s\n",
-	       DRMAA_REMOTE_COMMAND, error);
-    }
-    else {
-      const char *args[2] = {"10", NULL};
-      
-      errnum = drmaa_set_vector_attribute (jt, DRMAA_V_ARGV, args, error, \
-					   DRMAA_ERROR_STRING_BUFFER);
-    }
-    
-    if (errnum != DRMAA_ERRNO_SUCCESS) {
-      fprintf (stderr, "Could not set attribute \"%s\": %s\n",	\
-	       DRMAA_REMOTE_COMMAND, error);
-    }
-    else {
-      char jobid[DRMAA_JOBNAME_BUFFER];
-      
-      errnum = drmaa_run_job (jobid, DRMAA_JOBNAME_BUFFER, jt, error,	\
-			      DRMAA_ERROR_STRING_BUFFER);
-      
-      if (errnum != DRMAA_ERRNO_SUCCESS) {
-	fprintf (stderr, "Could not submit job: %s\n", error);
-      }
-      else {
-	printf ("Your job has been submitted with id %s\n", jobid);
-      }
-    } /* else */
-
-    errnum = drmaa_set_attribute (jt, DRMAA_REMOTE_COMMAND, "/bin/sleep", \
-				  error, DRMAA_ERROR_STRING_BUFFER);
-    
-    if (errnum != DRMAA_ERRNO_SUCCESS) {
-      fprintf (stderr, "Could not set attribute \"%s\": %s\n",
-	       DRMAA_REMOTE_COMMAND, error);
-    }
-    else {
-      const char *args[2] = {"12", NULL};
-      
-      errnum = drmaa_set_vector_attribute (jt, DRMAA_V_ARGV, args, error, \
-					   DRMAA_ERROR_STRING_BUFFER);
-    }
-    
-    if (errnum != DRMAA_ERRNO_SUCCESS) {
-      fprintf (stderr, "Could not set attribute \"%s\": %s\n",	\
-	       DRMAA_REMOTE_COMMAND, error);
-    }
-    else {
-      char jobid[DRMAA_JOBNAME_BUFFER];
-      
-      errnum = drmaa_run_job (jobid, DRMAA_JOBNAME_BUFFER, jt, error,	\
-			      DRMAA_ERROR_STRING_BUFFER);
-      
-      if (errnum != DRMAA_ERRNO_SUCCESS) {
-	fprintf (stderr, "Could not submit job: %s\n", error);
-      }
-      else {
-	printf ("Your job has been submitted with id %s\n", jobid);
-      }
-    } /* else */
-    
-    errnum = drmaa_delete_job_template (jt, error, DRMAA_ERROR_STRING_BUFFER);
-    
-    if (errnum != DRMAA_ERRNO_SUCCESS) {
-      fprintf (stderr, "Could not delete job template: %s\n", error);
-    }
-  } /* else */
-  
-  errnum = drmaa_exit (error, DRMAA_ERROR_STRING_BUFFER);
-  
-  if (errnum != DRMAA_ERRNO_SUCCESS) {
-	  fprintf (stderr, "Could not shut down the DRMAA library: %s\n", error);
-	  return 1;
-  }
-  printf ("DRMAA quit successfully\n");
-  return 0;
-}