Commits

Matthew Turk committed 64ee726

Beginning the consolidation of the --whatever argument parsing and moving a few
important items into a startup_items.py file. Command line utilities sort of
work but will not recognize or receive commands line --parallel.

Comments (0)

Files changed (6)

     __global_parallel_size = '1',
     __topcomm_parallel_rank = '0',
     __topcomm_parallel_size = '1',
+    __command_line = 'False',
     storeparameterfiles = 'False',
     parameterfilestore = 'parameter_files.csv',
     maximumstoredpfs = '500',
 from yt.utilities.logger import ytLogger as mylog
 import yt.utilities.progressbar as pb
 import yt.utilities.rpdb as rpdb
-
-# Some compatibility functions.  In the long run, these *should* disappear as
-# we move toward newer python versions.  Most were implemented to get things
-# running on DataStar.
-
-# If we're running on python2.4, we need a 'wraps' function
-def blank_wrapper(f):
-    return lambda a: a
-
-try:
-    from functools import wraps
-except ImportError:
-    wraps = blank_wrapper
-
-# We need to ensure that we have a defaultdict implementation
-
-class __defaultdict(dict):
-    def __init__(self, func):
-        self.__func = func
-        dict.__init__(self)
-    def __getitem__(self, key):
-        if not self.has_key(key):
-            self.__setitem__(key, self.__func())
-        return dict.__getitem__(self, key)
-
-try:
-    from collections import defaultdict
-except ImportError:
-    defaultdict = __defaultdict
+from collections import defaultdict
+from functools import wraps
 
 # Some functions for handling sequences and other types
 
     string to a list, for instance ensuring the *fields* as an argument is a
     list.
     """
-    if obj == None:
+    if obj is None:
         return [obj]
     if not isinstance(obj, types.ListType):
         return [obj]
 def signal_ipython(signo, frame):
     insert_ipython(2)
 
-# We use two signals, SIGUSR1 and SIGUSR2.  In a non-threaded environment,
-# we set up handlers to process these by printing the current stack and to
-# raise a RuntimeError.  The latter can be used, inside pdb, to catch an error
-# and then examine the current stack.
-try:
-    signal.signal(signal.SIGUSR1, signal_print_traceback)
-    mylog.debug("SIGUSR1 registered for traceback printing")
-    signal.signal(signal.SIGUSR2, signal_ipython)
-    mylog.debug("SIGUSR2 registered for IPython Insertion")
-except ValueError:  # Not in main thread
-    pass
-
 def paste_traceback(exc_type, exc, tb):
     """
     This is a traceback handler that knows how to paste to the pastebin.
     dec_s = ''.join([ chr(ord(a) ^ ord(b)) for a, b in zip(enc_s, itertools.cycle(key)) ])
     print dec_s
 
-# If we recognize one of the arguments on the command line as indicating a
-# different mechanism for handling tracebacks, we attach one of those handlers
-# and remove the argument from sys.argv.
-#
-# This fallback is for Paraview:
-if not hasattr(sys, 'argv') or sys.argv is None: sys.argv = []
-# Now, we check.
-if "--paste" in sys.argv:
-    sys.excepthook = paste_traceback
-    del sys.argv[sys.argv.index("--paste")]
-elif "--paste-detailed" in sys.argv:
-    sys.excepthook = paste_traceback_detailed
-    del sys.argv[sys.argv.index("--paste-detailed")]
-elif "--detailed" in sys.argv:
-    import cgitb; cgitb.enable(format="text")
-    del sys.argv[sys.argv.index("--detailed")]
-elif "--rpdb" in sys.argv:
-    sys.excepthook = rpdb.rpdb_excepthook
-    del sys.argv[sys.argv.index("--rpdb")]
-elif "--detailed" in sys.argv:
-    import cgitb; cgitb.enable(format="text")
-    del sys.argv[sys.argv.index("--detailed")]
-
 #
 # Some exceptions
 #
 import numpy as na # For historical reasons
 import numpy # In case anyone wishes to use it by name
 
+# This next item will handle most of the actual startup procedures, but it will
+# also attempt to parse the command line and set up the global state of various
+# operations.
+
+import yt.startup_tasks as __startup_tasks
+
 from yt.funcs import *
 from yt.utilities.logger import ytLogger as mylog
 from yt.utilities.performance_counters import yt_counters, time_function

yt/startup_tasks.py

+"""
+Very simple convenience function for importing all the modules, setting up
+the namespace and getting the last argument on the command line.
+
+Author: Matthew Turk <matthewturk@gmail.com>
+Affiliation: Columbia University
+Homepage: http://yt-project.org/
+License:
+  Copyright (C) 2011 Matthew Turk.  All Rights Reserved.
+
+  This file is part of yt.
+
+  yt is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+# This handles the command line.
+
+import argparse, os, sys
+
+from yt.config import ytcfg
+from yt.funcs import *
+
+exe_name = os.path.basename(sys.executable)
+# At import time, we determined whether or not we're being run in parallel.
+if exe_name in \
+        ["mpi4py", "embed_enzo",
+         "python"+sys.version[:3]+"-mpi"] \
+    or "--parallel" in sys.argv or '_parallel' in dir(sys) \
+    or any(["ipengine" in arg for arg in sys.argv]):
+    try:
+        from mpi4py import MPI
+        parallel_capable = (MPI.COMM_WORLD.size > 1)
+    except ImportError:
+        parallel_capable = False
+    if parallel_capable:
+        mylog.info("Global parallel computation enabled: %s / %s",
+                   MPI.COMM_WORLD.rank, MPI.COMM_WORLD.size)
+        ytcfg["yt","__global_parallel_rank"] = str(MPI.COMM_WORLD.rank)
+        ytcfg["yt","__global_parallel_size"] = str(MPI.COMM_WORLD.size)
+        ytcfg["yt","__parallel"] = "True"
+        if exe_name == "embed_enzo" or \
+            ("_parallel" in dir(sys) and sys._parallel == True):
+            ytcfg["yt","inline"] = "True"
+        # I believe we do not need to turn this off manually
+        #ytcfg["yt","StoreParameterFiles"] = "False"
+        # Now let's make sure we have the right options set.
+        if MPI.COMM_WORLD.rank > 0:
+            if ytcfg.getboolean("yt","LogFile"):
+                ytcfg["yt","LogFile"] = "False"
+                yt.utilities.logger.disable_file_logging()
+else:
+    parallel_capable = False
+
+# This fallback is for Paraview:
+
+# We use two signals, SIGUSR1 and SIGUSR2.  In a non-threaded environment,
+# we set up handlers to process these by printing the current stack and to
+# raise a RuntimeError.  The latter can be used, inside pdb, to catch an error
+# and then examine the current stack.
+try:
+    signal.signal(signal.SIGUSR1, signal_print_traceback)
+    mylog.debug("SIGUSR1 registered for traceback printing")
+    signal.signal(signal.SIGUSR2, signal_ipython)
+    mylog.debug("SIGUSR2 registered for IPython Insertion")
+except ValueError:  # Not in main thread
+    pass
+
+class SetExceptionHandling(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string = None):
+        # If we recognize one of the arguments on the command line as indicating a
+        # different mechanism for handling tracebacks, we attach one of those handlers
+        # and remove the argument from sys.argv.
+        #
+        if self.dest == "paste":
+            sys.excepthook = paste_traceback
+            mylog.debug("Enabling traceback pasting")
+        elif self.dest == "paste-detailed":
+            sys.excepthook = paste_traceback_detailed
+            mylog.debug("Enabling detailed traceback pasting")
+        elif self.dest == "detailed":
+            import cgitb; cgitb.enable(format="text")
+            mylog.debug("Enabling detailed traceback reporting")
+        elif self.dest == "rpdb":
+            sys.excepthook = rpdb.rpdb_excepthook
+            mylog.debug("Enabling remote debugging")
+
+class SetConfigOption(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string = None):
+        param, val = values.split("=")
+        mylog.debug("Overriding config: %s = %s", param, val)
+        ytcfg["yt",param] = val
+
+parser = argparse.ArgumentParser(description = 'yt command line arguments')
+parser.add_argument("--config", action=SetConfigOption,
+    help = "Set configuration option, in the form param=value")
+parser.add_argument("--paste", action=SetExceptionHandling,
+    help = "Paste traceback to paste.yt-project.org", nargs = 0)
+parser.add_argument("--paste-detailed", action=SetExceptionHandling,
+    help = "Paste a detailed traceback with local variables to " +
+           "paste.yt-project.org", nargs = 0)
+parser.add_argument("--detailed", action=SetExceptionHandling,
+    help = "Display detailed traceback.", nargs = 0)
+parser.add_argument("--rpdb", action=SetExceptionHandling,
+    help = "Enable remote pdb interaction (for parallel debugging).", nargs = 0)
+
+if not hasattr(sys, 'argv') or sys.argv is None: sys.argv = []
+
+if not ytcfg.getboolean("yt","__command_line"):
+    parser.parse_args()

yt/utilities/command_line.py

   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+from yt.config import ytcfg
+ytcfg["yt","__command_line"] = "True"
 from yt.mods import *
 from yt.funcs import *
 import cmdln as cmdln

yt/utilities/parallel_tools/parallel_analysis_interface.py

 from yt.utilities.amr_utils import \
     QuadTree, merge_quadtrees
 
-exe_name = os.path.basename(sys.executable)
-# At import time, we determined whether or not we're being run in parallel.
-if exe_name in \
-        ["mpi4py", "embed_enzo",
-         "python"+sys.version[:3]+"-mpi"] \
-    or "--parallel" in sys.argv or '_parallel' in dir(sys) \
-    or any(["ipengine" in arg for arg in sys.argv]):
+parallel_capable = ytcfg.getboolean("yt", "__parallel")
+
+# Set up translation table and import things
+if parallel_capable:
     from mpi4py import MPI
-    parallel_capable = (MPI.COMM_WORLD.size > 1)
-    if parallel_capable:
-        mylog.info("Global parallel computation enabled: %s / %s",
-                   MPI.COMM_WORLD.rank, MPI.COMM_WORLD.size)
-        ytcfg["yt","__global_parallel_rank"] = str(MPI.COMM_WORLD.rank)
-        ytcfg["yt","__global_parallel_size"] = str(MPI.COMM_WORLD.size)
-        ytcfg["yt","__parallel"] = "True"
-        if exe_name == "embed_enzo" or \
-            ("_parallel" in dir(sys) and sys._parallel == True):
-            ytcfg["yt","inline"] = "True"
-        # I believe we do not need to turn this off manually
-        #ytcfg["yt","StoreParameterFiles"] = "False"
-        # Now let's make sure we have the right options set.
-        if MPI.COMM_WORLD.rank > 0:
-            if ytcfg.getboolean("yt","LogFile"):
-                ytcfg["yt","LogFile"] = "False"
-                yt.utilities.logger.disable_file_logging()
-        yt.utilities.logger.uncolorize_logging()
-        # Even though the uncolorize function already resets the format string,
-        # we reset it again so that it includes the processor.
-        f = logging.Formatter("P%03i %s" % (MPI.COMM_WORLD.rank,
-                                            yt.utilities.logger.ufstring))
-        if len(yt.utilities.logger.rootLogger.handlers) > 0:
-            yt.utilities.logger.rootLogger.handlers[0].setFormatter(f)
-        if ytcfg.getboolean("yt", "parallel_traceback"):
-            sys.excepthook = traceback_writer_hook("_%03i" % MPI.COMM_WORLD.rank)
+    yt.utilities.logger.uncolorize_logging()
+    # Even though the uncolorize function already resets the format string,
+    # we reset it again so that it includes the processor.
+    f = logging.Formatter("P%03i %s" % (MPI.COMM_WORLD.rank,
+                                        yt.utilities.logger.ufstring))
+    if len(yt.utilities.logger.rootLogger.handlers) > 0:
+        yt.utilities.logger.rootLogger.handlers[0].setFormatter(f)
+    if ytcfg.getboolean("yt", "parallel_traceback"):
+        sys.excepthook = traceback_writer_hook("_%03i" % MPI.COMM_WORLD.rank)
     if ytcfg.getint("yt","LogLevel") < 20:
         yt.utilities.logger.ytLogger.warning(
           "Log Level is set low -- this could affect parallel performance!")
-
-else:
-    parallel_capable = False
-
-# Set up translation table
-if parallel_capable:
     dtype_names = dict(
             float32 = MPI.FLOAT,
             float64 = MPI.DOUBLE,
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.