Commits

Johannes Köster  committed 86e4328 Merge

Merge branch 'improved_exception_handling'

  • Participants
  • Parent commits 27e6f95, 9a1431f

Comments (0)

Files changed (8)

File snakemake/__init__.py

     parser.add_argument(
         "--debug", action="store_true", help="Print debugging output.")
     parser.add_argument(
+        "--profile", help="Profile Snakemake. This requires yappi to be installed.")
+    parser.add_argument(
             "--bash-completion", action="store_true", help="Output code to register bash completion for snakemake. Put the following in your .bashrc (including the accents): `snakemake --bash-completion`")
     parser.add_argument(
         "--version", "-v", action="version", version=__version__)
         parser.print_help()
         sys.exit(1)
 
+    if args.profile:
+        import yappi
+        yappi.start()
+
     success = snakemake(
             args.snakefile,
             listrules=args.list,
             jobscript=args.jobscript,
             notemp=args.notemp,
             timestamp=args.timestamp)
+
+    if args.profile:
+        with open(args.profile, "w") as out:
+            profile = yappi.get_func_stats()
+            profile.sort("totaltime")
+            profile.print_all(out=out)
+
     sys.exit(0 if success else 1)
 
 

File snakemake/dag.py

                 if file in job.dependencies:
                     jobs = [Job(job.dependencies[file], self, targetfile=file)]
                 else:
-                    jobs = self.file2jobs(file)
+                    jobs = file2jobs(file)
                 dependencies[file].extend(jobs)
             except MissingRuleException as ex:
                 pass
     def rule2job(self, targetrule):
         return Job(targetrule, self)
 
+    @lru_cache()
     def file2jobs(self, targetfile):
         jobs = [Job(rule, self, targetfile=targetfile) for rule in self.rules if rule.is_producer(targetfile)]
         if not jobs:

File snakemake/executors.py

                 raise ex
             self.finish_job(job)
             callback(job)
+        except KeyboardInterrupt:
+            job.cleanup()
+            self.workflow.persistence.cleanup(job)
+            # no error callback, just silently ignore the interrupt as the main scheduler is also killed
         except (Exception, BaseException) as ex:
             print_exception(ex, self.workflow.linemaps)
             job.cleanup()
     try:
         # execute the actual run method.
         run(input, output, params, wildcards, threads, resources, log)
+    except KeyboardInterrupt as e:
+        # re-raise the keyboard interrupt in order to record an error in the scheduler but ignore it
+        raise e
     except (Exception, BaseException) as ex:
         # this ensures that exception can be re-raised in the parent thread
         lineno, file = get_exception_origin(ex, linemaps)

File snakemake/io.py

         return contains_wildcard(self.file)
 
     def regex(self):
-        if not self._regex:
+        if self._regex is None:
             # compile a regular expression
             self._regex = re.compile(regex(self.file))
         return self._regex
 
     def match(self, target):
-        match = self.regex().match(target)
-        return match if match else None
+        return self.regex().match(target) or None
 
     def __eq__(self, other):
         f = other._file if isinstance(other, _IOFile) else other

File snakemake/jobs.py

 __author__ = "Johannes Köster"
 
 
+def jobfiles(jobs, type):
+    return chain(*map(attrgetter(type), jobs))
+
+
 class Job:
     HIGHEST_PRIORITY = sys.maxsize
 
-    @staticmethod
-    def files(jobs, type):
-        return chain(*map(attrgetter(type), jobs))
-
     def __init__(self, rule, dag, targetfile=None, format_wildcards=None):
         self.rule = rule
         self.dag = dag
         self.targetfile = targetfile
+
         self.wildcards_dict = self.rule.get_wildcards(targetfile)
         self.wildcards = Wildcards(fromdict=self.wildcards_dict)
         self._format_wildcards = (self.wildcards
     def __gt__(self, other):
         return self.rule.__gt__(other.rule)
 
-
     def __hash__(self):
         return self._hash
 

File snakemake/persistence.py

 from itertools import filterfalse, count
 
 from snakemake.logging import logger
-from snakemake.jobs import Job
+from snakemake.jobs import jobfiles
 from snakemake.utils import listfiles
 
 
 
     def outputfiles(self):
         # we only look at output files that will be updated
-        return Job.files(self.dag.needrun_jobs, "output")
+        return jobfiles(self.dag.needrun_jobs, "output")
 
     def inputfiles(self):
         # we consider all input files, also of not running jobs
-        return Job.files(self.dag.jobs, "input")
+        return jobfiles(self.dag.jobs, "input")
 

File snakemake/rules.py

 
         if wildcards is None:
             wildcards = dict()
-        # TODO validate
         missing_wildcards = self.wildcard_names - set(wildcards.keys())
 
         if missing_wildcards:

File snakemake/scheduler.py

 
     def schedule(self):
         """ Schedule jobs that are ready, maximizing cpu usage. """
-        
         while True:
             try:
-                self._open_jobs.wait()
+                # work around so that the wait does not prevent keyboard interrupts
+                while not self._open_jobs.wait(1): pass
             except:
                 # this will be caused because of SIGTERM or SIGINT
                 logger.info("Terminating processes on user request.")