Commits

Johannes Köster committed d6ffd3a

Support for workflow config.

Comments (0)

Files changed (7)

snakemake/__init__.py

     cores=1,
     nodes=1,
     resources=dict(),
+    config=dict(),
     workdir=None,
     targets=None,
     dryrun=False,
         cores (int):                the number of provided cores (ignored when using cluster support) (default 1)
         nodes (int):                the number of provided cluster nodes (ignored without cluster support) (default 1)
         resources (dict):           provided resources, a dictionary assigning integers to resource names, e.g. {gpu=1, io=5} (default {})
+        config (dict):              override values for workflow config
         workdir (str):              path to working directory (default None)
         targets (list):             list of targets, e.g. rule or file names (default None)
         dryrun (bool):              only dry-run the workflow (default False)
         olddir = os.getcwd()
     workflow = Workflow(
         snakefile=snakefile, snakemakepath=snakemakepath,
-        jobscript=jobscript, overwrite_shellcmd=overwrite_shellcmd)
+        jobscript=jobscript, overwrite_shellcmd=overwrite_shellcmd,
+        overwrite_config=config)
 
     if standalone:
         try:
     return resources
 
 
+def parse_config(args):
+    parsers = [int, float, eval, str]
+    config = dict()
+    if args.config is not None:
+        valid = re.compile("[a-zA-Z_]\w*$")
+        for entry in args.config:
+            try:
+                key, val = entry.split("=", maxsplit=1)
+            except ValueError:
+                raise ValueError("Config entries have to be defined as name=value pairs.")
+            if not valid.match(key):
+                raise ValueError("Config entry must start with a valid identifier.")
+            v = None
+            for parser in parsers:
+                try:
+                    v = parser(val)
+                    break
+                except:
+                    pass
+            assert v is not None
+            config[key] = v
+    return config
+
+
 def get_argument_parser():
     parser = argparse.ArgumentParser(
         description="Snakemake is a Python based language and execution "
             "resources: gpu=1. If now two rules require 1 of the resource "
             "'gpu' they won't be run in parallel by the scheduler."))
     parser.add_argument(
+        "--config", nargs="*", metavar="KEY=VALUE",
+        help=(
+            "Set or overwrite values in the workflow config object. "
+            "The workflow config object is accessible as variable config inside "
+            "the workflow. Default values can be set by providing a JSON file "
+            "(see Documentation)."))
+    parser.add_argument(
         "--list", "-l", action="store_true",
         help="Show availiable rules in given Snakefile.")
     parser.add_argument(
 
     try:
         resources = parse_resources(args)
+        config = parse_config(args)
     except ValueError as e:
         print(e, file=sys.stderr)
         print("", file=sys.stderr)
             cores=args.cores,
             nodes=args.cores,
             resources=resources,
+            config=config,
             workdir=args.directory,
             targets=args.target,
             dryrun=args.dryrun,

snakemake/parser.py

     pass
 
 
+class Config(GlobalKeywordState):
+    pass
+
+
 class Ruleorder(GlobalKeywordState):
 
     def block_content(self, token):
     subautomata = dict(
         include=Include,
         workdir=Workdir,
+        config=Config,
         ruleorder=Ruleorder,
         rule=Rule,
         subworkflow=Subworkflow,
         for t, orig_token in automaton.consume():
             l = lineno(orig_token)
             linemap.update(
-                dict((i, l) for i in range(snakefile.lines, snakefile.lines + t.count("\n"))))
+                dict((i, l) for i in range(snakefile.lines + 1, snakefile.lines + t.count("\n") + 1)))
             snakefile.lines += t.count("\n")
             compilation.append(t)
         compilation = "".join(format_tokens(compilation))

snakemake/workflow.py

 import os
 import sys
 import signal
+import json
 from collections import OrderedDict
 from itertools import filterfalse, chain
 from functools import partial
 from snakemake.logging import logger, format_resources, format_resource_names
 from snakemake.rules import Rule, Ruleorder
 from snakemake.exceptions import RuleException, CreateRuleException, \
-    UnknownRuleException, NoRulesException, print_exception
+    UnknownRuleException, NoRulesException, print_exception, WorkflowError
 from snakemake.shell import shell
 from snakemake.dag import DAG
 from snakemake.scheduler import JobScheduler
 class Workflow:
     def __init__(
         self, snakefile=None, snakemakepath=None,
-        jobscript=None, overwrite_shellcmd=None):
+        jobscript=None, overwrite_shellcmd=None,
+        overwrite_config=dict()):
         """
         Create the controller.
         """
         self.globals = globals()
         self._subworkflows = dict()
         self.overwrite_shellcmd = overwrite_shellcmd
+        self.overwrite_config = overwrite_config
+
+        global config
+        config = dict()
+        config.update(self.overwrite_config)
 
     @property
     def subworkflows(self):
                 os.makedirs(workdir)
             self._workdir = workdir
 
+    def config(self, jsonpath):
+        """ Update the global config with the given dictionary. """
+        global config
+        try:
+            with open(jsonpath) as f:
+                c = json.load(f)
+        except FileNotFoundError:
+            raise WorkflowError("Config file {} not found.".format(jsonpath))
+        if not isinstance(c, dict):
+            raise WorkflowError("Workflow config must be given as JSON with keys at top level.")
+        for key, val in c.items():
+            if key not in self.overwrite_config:
+                config[key] = val
+
     def ruleorder(self, *rulenames):
         self._ruleorder.add(*rulenames)
 

tests/test_config/Snakefile

+
+config: "test.json"
+
+
+rule:
+    output:
+        config["outfile"]
+    shell:
+        "touch {output}"

tests/test_config/expected-results/test.out

Empty file added.

tests/test_config/test.json

+{
+    "outfile": "test.out"
+}
 
 def test_touch():
     run(dpath("test_touch"))
+
+def test_config():
+    run(dpath("test_config"))