Commits

Yung-Yu Chen committed bf86ddb

Add conda installation and stop/terminate all.

  • Participants
  • Parent commits c2727ac

Comments (0)

Files changed (2)

File solvcon/cloud.py

 
 import os
 import collections
+import functools
 
 import boto.ec2
 import paramiko as ssh
 import helper
 
 
-__all__ = ['AwsHost']
+__all__ = ['AwsHost', 'ahsregy']
 
 
 class OperatingSystem(object):
         return self._MINPKGS[self.osfamily]
 
 
-class AwsHostRegistry(dict):
+class AwsHostSettingRegistry(dict):
     @classmethod
     def populate(cls):
         regy = cls()
-        regy["RHEL64"] = AwsHostSetting(region="us-west-2", ami="ami-77d7a747",
-                                        osfamily="RHEL", username="ec2-user")
+        regy["RHEL64"] = AwsHostSetting(
+            region="us-west-2", ami="ami-77d7a747",
+            osfamily="RHEL", username="ec2-user")
         regy["trusty64"] = AwsHostSetting(
             region="us-west-2", ami="ami-d34032e3",
             osfamily="Ubuntu", username="ubuntu")
         regy[""] = regy["trusty64"]
         return regy
 
-ahregy = AwsHostRegistry.populate()
+ahsregy = AwsHostSettingRegistry.populate()
 
 
 class AwsHost(object):
     Abstraction for actions toward an AWS EC2 host.
     """
 
+    MINICONDA_URL = ("http://repo.continuum.io/miniconda/"
+                     "Miniconda-3.5.5-Linux-x86_64.sh")
+
     def __init__(self, instance, setting):
         #: :py:class:`boto.ec2.instance.Instance` for the host.
         self.instance = instance
         self.cli.disconnect()
         self.cli = None
 
-    def run(self, cmd, sudo=False, wd=None, bufsize=-1, timeout=None):
+    @staticmethod
+    def _prepare_command(cmd, sudo=False, env=None, wd=None):
+        """
+        This is the helper method to make up the command (*cmd*) with different
+        settings.
+
+        The optional argument *sudo* will prefix "sudo".  Default is False:
+
+        >>> AwsHost._prepare_command("cmd")
+        'cmd'
+        >>> AwsHost._prepare_command("cmd", sudo=True)
+        'sudo cmd'
+
+        The optional argument *env* will prefix "env" with the given string of
+        environment variables.  The default is None:
+
+        >>> AwsHost._prepare_command("cmd", env="PATH=$HOME:$PATH")
+        'env PATH=$HOME:$PATH cmd'
+
+        The optional argument *wd* will first change the working directory and
+        execute the command.  The default is None:
+
+        >>> AwsHost._prepare_command("cmd", wd="/tmp")
+        'cd /tmp; cmd'
+
+        Argument *env* can be used with either *sudo* or *wd*:
+
+        >>> AwsHost._prepare_command("cmd", sudo=True,
+        ...                          env="PATH=$HOME:$PATH")
+        'sudo env PATH=$HOME:$PATH cmd'
+        >>> AwsHost._prepare_command("cmd", env="PATH=$HOME:$PATH", wd="/tmp")
+        'cd /tmp; env PATH=$HOME:$PATH cmd'
+
+        However, *sudo* doesn't work with *wd*:
+
+        >>> AwsHost._prepare_command("cmd", sudo=True, wd="/tmp")
+        Traceback (most recent call last):
+            ...
+        ValueError: sudo can't be True with wd set
+        """
+        if env is not None:
+            cmd = "env %s %s" % (env, cmd)
         if wd is not None:
             cmd = "cd %s; %s" % (wd, cmd)
-        helper.info(cmd + "\n")
+            if sudo:
+                raise ValueError("sudo can't be True with wd set")
+        if sudo:
+            cmd = "sudo %s" % cmd
+        return cmd
+
+    @staticmethod
+    def _setup_channel(chan, bufsize=-1, timeout=None):
         chan = self.cli.get_transport().open_session()
         forward = ssh.agent.AgentRequestHandler(chan)
         chan.get_pty()
         chan.set_combine_stderr(True)
         chan.settimeout(timeout)
-        if sudo:
-            cmd = "sudo %s" % cmd
         chan.exec_command(cmd)
         stdin = chan.makefile('wb', bufsize)
         stdout = chan.makefile('r', bufsize)
         data = stdout.read()
+
+    def run(self, cmd, bufsize=-1, timeout=None, **kw):
+        # Prepare the command.
+        cmd = self._prepare_command(cmd, **kw)
+        # Log command information.
+        helper.info(cmd + "\n")
+        # Open the channel.
+        chan = self.cli.get_transport().open_session()
+        # Set up SSH authentication agent forwarding.
+        forward = ssh.agent.AgentRequestHandler(chan)
+        # Get and set up the terminal.
+        chan.get_pty()
+        chan.set_combine_stderr(True)
+        chan.settimeout(timeout)
+        # Send the command.
+        chan.exec_command(cmd)
+        # Use the STD I/O.
+        stdin = chan.makefile('wb', bufsize)
+        stdout = chan.makefile('r', bufsize)
+        # Get the data and report.
+        data = stdout.read()
         helper.info(data + "\n")
         return data
 
             packages = " ".join(packages)
         self.run("%s %s" % (manager, packages), sudo=True)
 
-    def install_minimal(self):
-        self.install(self.setting.minpkgs)
-
     def hgclone(self, path, sshcmd="", **kw):
         cmd = "hg clone"
         if path.startswith("ssh") and sshcmd:
             cmd += " --ssh '%s'" % sshcmd
         cmd = "%s %s" % (cmd, path)
         self.run(cmd, **kw)
+
+    def deploy_minimal(self):
+        # Use OS package manager to install tools.
+        self.install(self.setting.minpkgs)
+        # Install miniconda.
+        mcurl = self.MINICONDA_URL
+        mcfn = mcurl.split("/")[-1]
+        run = functools.partial(self.run, wd="/tmp")
+        run("rm -f %s" % mcfn)
+        run("wget %s" % mcurl)
+        run("bash %s -p ~/opt/miniconda -b" % mcfn)
+        # Update and install conda packages.
+        run = functools.partial(self.run,
+                                env="PATH=$HOME/opt/miniconda/bin:$PATH")
+        run("conda update --all --yes")
+        run("conda install jinja2 --yes")
+        run("conda install setuptools mercurial conda-build "
+            "scons cython numpy netcdf4 nose sphinx paramiko boto "
+            "--yes")

File solvcon/command.py

         instkws = ("region", "ami", "username",
                    "instance_type", "security_groups")
         instkws = dict((key, None) for key in instkws)
-        ahs = cloud.ahregy[ops.regname]
+        ahs = cloud.ahsregy[ops.regname]
         update_kws(instkws, ahs)
         update_kws(instkws, ops)
         sg = instkws["security_groups"]
                 else:
                     helper.info("Connection refused.\n")
                     raise
-        host.install_minimal()
+        host.deploy_minimal()
 
         if args:
             self.run_script(host, args[0])
     Stop AWS EC2 instance.
     """
 
-    min_args = 1
+    min_args = 0
 
     def __init__(self, env):
         from optparse import OptionGroup
         op = self.op
         opg = OptionGroup(op, 'AWS Stop')
         opg.add_option(
+            "--all", action="store_true", default=False,
+            help="Terminate all (default: \"%(default)s\").")
+        opg.add_option(
             "--force", action="store_true", default=False,
             help="Force action (default: \"%(default)s\").")
         op.add_option_group(opg)
         conn = boto.ec2.connect_to_region(
             ops.region, aws_access_key_id=ops.access_key_id,
             aws_secret_access_key=ops.secret_access_key)
-        instances = conn.get_only_instances(instance_ids=args)
+        if ops.all:
+            reservations = conn.get_all_reservations()
+            instances = itertools.chain(*(r.instances for r in reservations))
+            instances = list(instances)
+        else:
+            instances = conn.get_only_instances(instance_ids=args)
         for inst in instances:
             status = inst.update()
-            msgs = self.instance_information(inst, status)
-            helper.info("Stopping " + "\n  ".join(msgs) + "\n")
-            inst.stop(force=ops.force)
+            if status not in ("stopped", "terminated"):
+                msgs = self.instance_information(inst, status)
+                helper.info("Stopping " + "\n  ".join(msgs) + "\n")
+                inst.stop(force=ops.force)
 
 
 class aterminate(Aws):
     Terminate AWS EC2 instance.
     """
 
-    min_args = 1
+    min_args = 0
+
+    def __init__(self, env):
+        from optparse import OptionGroup
+        super(aterminate, self).__init__(env)
+        op = self.op
+        opg = OptionGroup(op, 'AWS Terminate')
+        opg.add_option(
+            "--all", action="store_true", default=False,
+            help="Terminate all (default: \"%(default)s\").")
+        op.add_option_group(opg)
+        self.opg_aws_stop = opg
 
     def __call__(self):
         ops, args = self.opargs
         conn = boto.ec2.connect_to_region(
             ops.region, aws_access_key_id=ops.access_key_id,
             aws_secret_access_key=ops.secret_access_key)
-        instances = conn.get_only_instances(instance_ids=args)
+        if ops.all:
+            reservations = conn.get_all_reservations()
+            instances = itertools.chain(*(r.instances for r in reservations))
+            instances = list(instances)
+        else:
+            instances = conn.get_only_instances(instance_ids=args)
         for inst in instances:
             status = inst.update()
-            msgs = self.instance_information(inst, status)
-            helper.info("Terminating " + "\n  ".join(msgs) + "\n")
-            inst.terminate()
+            if status != "terminated":
+                msgs = self.instance_information(inst, status)
+                helper.info("Terminating " + "\n  ".join(msgs) + "\n")
+                inst.terminate()
 
 
 class mpi(Command):