Commits

Anonymous committed da25504

Implement merged messages support

This version supports merging of output and error messages as an
alternative reporting style. This can be used with --progress

  • Participants
  • Parent commits bdfc3f1
  • Tags 0.091

Comments (0)

Files changed (3)

 __author__  = '''Jim Dennis <answrguy@gmail.com>'''
 __url__     = '''http://bitbucket.org/jimd/classh/'''
 __license___= '''PSF (Python Software Foundation)'''
-__version__ = 0.08
+__version__ = 0.09
 
 
 import sys, os, string, signal, re
 
         self.pool    = dict()         # for maintaining host/process data
         self.results = dict()  
+
+        self.mergeout = dict()
+        self.mergeerr = dict()
+        
         self.ssh = [ '/usr/bin/ssh' ]        #TODO: dynamically find this
         self.ssh_args = [ '-oBatchMode=yes'] #TODO: add other batch options?
 
         '''
         debug ('gathering')
         try: 
-            (job.output, job.errors) = job.proc.communicate()
+            (output, errors) = job.proc.communicate()
         except ValueError, e: # Popen error on short timeouts?
             debug ('Popen.communicate exception')
-            (job.output, job.errors) = ('','')
-            job.exitcode = -1000  ##TODO: Popen:Exception
+            (output, errors) = ('','')
+            job.exitcode = - (1000 + job.exitcode) 
+            ##TODO: Popen:Exception
         job.stopped  = time()
+
+        if output.strip() == '':  # If it's all whitespace
+            output = ''           # distill it down to nothing
+
+        if errors.strip() == '':  # ditto
+            errors = ''
+
+        # Keep messages merged and possibly force intern of results
+        if output in self.mergeout:
+            self.mergeout[output].append(key)
+        else:
+            self.mergeout[output] = [key]
+
+        if errors in self.mergeerr:
+            self.mergeerr[errors].append(key)
+        else:
+            self.mergeerr[errors] = [key]
+
+        job.output= output
+        job.errors= errors
+
         self.results[key] = job
+
             
     def kill(self, proc):
         '''Kill a subprocess which has exceeded its timeout
         debug ('wait called called with: %s %s ' % (maxwait,snooze))
         return_after=None
         if maxwait is not None:
+            # Pre-calculation return time:
             return_after = time() + maxwait
-        if snooze is not None:
+        if snooze is None:
             snooze = self.sleeptime
         if not self.started:
             self.start()
            '.'*len(string.translate(string.maketrans('',''),
            string.maketrans('',''),string.printable[:-5])))
 
+# Use to munge non-printable while preserving tabs and newlines
+lfilter = string.maketrans(
+           string.translate(string.maketrans('',''),
+           string.maketrans('',''),string.printable[:-3]),
+           '.'*len(string.translate(string.maketrans('',''),
+           string.maketrans('',''),string.printable[:-3])))
+
 
 def print_results(key, job):
     errs = ' '.join(job.errors.split('\n'))
     elif  job.exitcode   <      0: res = '!'
     sys.stderr.write(res)   # avoid trailing spaciness from print statement
             
-def summarize_results(res):
+
+def summarize_results(job):
+    '''Print the number of success, errors, and timeouts from a job
+    '''
     success  = 0
     errs     = 0
     timeouts = 0
-    for i in res.values():
+    for i in job.values():
         if    i.exitcode  ==  0: success  += 1
         elif  i.exitcode   >  0: errs     += 1
         elif  i.exitcode   <  0: timeouts += 1
     print "\n\n\tSuccessful: %s\tErrors: %s\tSignaled(timeouts): %s" \
       % (success, errs, timeouts)
 
+
+def comma_join(lst):
+    '''Print list in a natural way, joined by commas with the last
+       after an ", and " 
+    '''
+    if len(lst) < 2:
+        return lst[0]
+    return ', '.join(lst[:-1]) + ', and ' + lst[-1]
+
+
+
+def summarize_merged_results(jhand):
+    '''Given a job handle (object) print each unique output or error message
+       along with the list of hosts from which it was gathered.
+
+       Separately print the number of distinct messages and the number
+       of successful and error-returning jobs with no messages.
+    '''
+    success  = 0
+    errs     = 0
+    timeouts = 0
+
+    print "\n"  # Ensure a clear line, esp. if progressbar enabled
+    for out in jhand.mergeout.keys():
+        if out == '':
+            continue 
+        n = len(jhand.mergeout[out])
+        print "Output from (%s hosts): " % n, comma_join(jhand.mergeout[out])
+        print "============================================================="
+        print out.translate(lfilter)
+        print "=============================================================\n"
+
+    for err in jhand.mergeerr.keys():
+        if err == '':
+            continue 
+        n = len(jhand.mergeerr[err])
+        print "Errors from (%s hosts): " % n, comma_join(jhand.mergeerr[err])
+        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        print err.translate(lfilter)
+        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+
+    for i in jhand.results.values():
+        if    i.exitcode  ==  0: success  += 1
+        elif  i.exitcode   >  0: errs     += 1
+        elif  i.exitcode   <  0: timeouts += 1
+    print "\n\n\tSuccessful: %s\tErrors: %s\tSignaled(timeouts): %s" \
+      % (success, errs, timeouts)
+
+
+
 def readfile(fname):
     results = []
     try: 
         f = open(fname)
     except EnvironmentError, e:
+        ##TODO: logger! 
         print >> sys.stderr, "Error reading %s, %s" % (fname, e)
     for line in f:
-        if line.endswith('\n'):
-            line = line[:-1].strip()
-        results.append(line)
+        results.append(line.strip())
     return results
 
 usage = '''
        '--noexpand', dest='noexpand', 
        action='store_true', default=False,
        help="Disable expansion of hostname ranges")
+
+    parser.add_option(
+       '--mergemsgs', dest='mergemsgs', 
+       action='store_true', default=False,
+       help="Merge identical output/error messages")
        
     opts, args = parser.parse_args()
     return (parser, opts, args)
 
 if __name__ == "__main__":
-    '''Stupid simple test code and wrapper:
-       First argument is command, rest are hosts.
+    '''First argument is command, rest are hosts.
        Dispatch jobs to hosts and incrementally print results
+       or progress bar
     '''
 
     start = time()
     hosts = list()
 
     handle_completed = print_results
+    if options.mergemsgs:
+        handle_completed = lambda x,y : None
     if options.progressbar:
         handle_completed = show_progress
     
             for each in completed:
                 handle_completed(each, job.results[each])
         
-    summarize_results(job.results)
+    if not options.mergemsgs:    
+        summarize_results(job.results)
+    else:
+        summarize_merged_results(job)
+    
     print "Completed in %s seconds" % (time() - start)
 
 
 __author__  = '''Jim Dennis <answrguy@gmail.com>'''
 __url__     = '''http://bitbucket.org/jimd/classh/'''
 __license___= '''PSF (Python Software Foundation)'''
-__version__ = 0.08
+__version__ = 0.09
 
 
 import sys, os, string, signal, re
 
         self.pool    = dict()         # for maintaining host/process data
         self.results = dict()  
+
+        self.mergeout = dict()
+        self.mergeerr = dict()
+        
         self.ssh = [ '/usr/bin/ssh' ]        #TODO: dynamically find this
         self.ssh_args = [ '-oBatchMode=yes'] #TODO: add other batch options?
 
         '''
         debug ('gathering')
         try: 
-            (job.output, job.errors) = job.proc.communicate()
+            (output, errors) = job.proc.communicate()
         except ValueError, e: # Popen error on short timeouts?
             debug ('Popen.communicate exception')
-            (job.output, job.errors) = ('','')
-            job.exitcode = -1000  ##TODO: Popen:Exception
+            (output, errors) = ('','')
+            job.exitcode = - (1000 + job.exitcode) 
+            ##TODO: Popen:Exception
         job.stopped  = time()
+
+        if output.strip() == '':  # If it's all whitespace
+            output = ''           # distill it down to nothing
+
+        if errors.strip() == '':  # ditto
+            errors = ''
+
+        # Keep messages merged and possibly force intern of results
+        if output in self.mergeout:
+            self.mergeout[output].append(key)
+        else:
+            self.mergeout[output] = [key]
+
+        if errors in self.mergeerr:
+            self.mergeerr[errors].append(key)
+        else:
+            self.mergeerr[errors] = [key]
+
+        job.output= output
+        job.errors= errors
+
         self.results[key] = job
+
             
     def kill(self, proc):
         '''Kill a subprocess which has exceeded its timeout
         debug ('wait called called with: %s %s ' % (maxwait,snooze))
         return_after=None
         if maxwait is not None:
+            # Pre-calculation return time:
             return_after = time() + maxwait
         if snooze is None:
             snooze = self.sleeptime
            '.'*len(string.translate(string.maketrans('',''),
            string.maketrans('',''),string.printable[:-5])))
 
+# Use to munge non-printable while preserving tabs and newlines
+lfilter = string.maketrans(
+           string.translate(string.maketrans('',''),
+           string.maketrans('',''),string.printable[:-3]),
+           '.'*len(string.translate(string.maketrans('',''),
+           string.maketrans('',''),string.printable[:-3])))
+
 
 def print_results(key, job):
     errs = ' '.join(job.errors.split('\n'))
     elif  job.exitcode   <      0: res = '!'
     sys.stderr.write(res)   # avoid trailing spaciness from print statement
             
-def summarize_results(res):
+
+def summarize_results(job):
+    '''Print the number of success, errors, and timeouts from a job
+    '''
     success  = 0
     errs     = 0
     timeouts = 0
-    for i in res.values():
+    for i in job.values():
         if    i.exitcode  ==  0: success  += 1
         elif  i.exitcode   >  0: errs     += 1
         elif  i.exitcode   <  0: timeouts += 1
     print "\n\n\tSuccessful: %s\tErrors: %s\tSignaled(timeouts): %s" \
       % (success, errs, timeouts)
 
+
+def comma_join(lst):
+    '''Print list in a natural way, joined by commas with the last
+       after an ", and " 
+    '''
+    if len(lst) < 2:
+        return lst[0]
+    return ', '.join(lst[:-1]) + ', and ' + lst[-1]
+
+
+
+def summarize_merged_results(jhand):
+    '''Given a job handle (object) print each unique output or error message
+       along with the list of hosts from which it was gathered.
+
+       Separately print the number of distinct messages and the number
+       of successful and error-returning jobs with no messages.
+    '''
+    success  = 0
+    errs     = 0
+    timeouts = 0
+
+    print "\n"  # Ensure a clear line, esp. if progressbar enabled
+    for out in jhand.mergeout.keys():
+        if out == '':
+            continue 
+        n = len(jhand.mergeout[out])
+        print "Output from (%s hosts): " % n, comma_join(jhand.mergeout[out])
+        print "============================================================="
+        print out.translate(lfilter)
+        print "=============================================================\n"
+
+    for err in jhand.mergeerr.keys():
+        if err == '':
+            continue 
+        n = len(jhand.mergeerr[err])
+        print "Errors from (%s hosts): " % n, comma_join(jhand.mergeerr[err])
+        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        print err.translate(lfilter)
+        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+
+    for i in jhand.results.values():
+        if    i.exitcode  ==  0: success  += 1
+        elif  i.exitcode   >  0: errs     += 1
+        elif  i.exitcode   <  0: timeouts += 1
+    print "\n\n\tSuccessful: %s\tErrors: %s\tSignaled(timeouts): %s" \
+      % (success, errs, timeouts)
+
+
+
 def readfile(fname):
     results = []
     try: 
         f = open(fname)
     except EnvironmentError, e:
+        ##TODO: logger! 
         print >> sys.stderr, "Error reading %s, %s" % (fname, e)
     for line in f:
-        if line.endswith('\n'):
-            line = line[:-1].strip()
-        results.append(line)
+        results.append(line.strip())
     return results
 
 usage = '''
        '--noexpand', dest='noexpand', 
        action='store_true', default=False,
        help="Disable expansion of hostname ranges")
+
+    parser.add_option(
+       '--mergemsgs', dest='mergemsgs', 
+       action='store_true', default=False,
+       help="Merge identical output/error messages")
        
     opts, args = parser.parse_args()
     return (parser, opts, args)
 
 if __name__ == "__main__":
-    '''Stupid simple test code and wrapper:
-       First argument is command, rest are hosts.
+    '''First argument is command, rest are hosts.
        Dispatch jobs to hosts and incrementally print results
+       or progress bar
     '''
 
     start = time()
     hosts = list()
 
     handle_completed = print_results
+    if options.mergemsgs:
+        handle_completed = lambda x,y : None
     if options.progressbar:
         handle_completed = show_progress
     
             for each in completed:
                 handle_completed(each, job.results[each])
         
-    summarize_results(job.results)
+    if not options.mergemsgs:    
+        summarize_results(job.results)
+    else:
+        summarize_merged_results(job)
+    
     print "Completed in %s seconds" % (time() - start)
 
 
 
 setup(
     name='classh',
-    version='0.08',
+    version='0.09',
     author='James T. Dennis',
     author_email='answrguy@gmail.com',
     license='PSF',