Source

stagehand / install / tools.py

"""
stagehand.install.tools

Utilities to install a package from a repository, or failing that, 
from source. It does not do any dependency management (other than
what is built into apt), so make sure that any necessary 
dependencies are installed first. Requires a working fabric 
installation on the installing host.

Copyright 2010-2011 Van Lindberg. Released under the BSD license.

The exported API consists of the following functions:

- update_apt(): Updates the list of packages available to apt.
  Exposed as a task.

  update_packages(): Updates existing installed packages using apt.
  Exposed as a task.
  
  add_apt_repository(repo): Runs add-apt-repository to add the 
  requested repo. Exposed as a task.
  
  check_installed(package): Uses which, dpkg, and locate (if 
  available) to see if the requested package is installed. This 
  is a heuristic and can be fooled, so be aware. Some packages 
  may need a better or more specific method of testing 
  installation. Returns True or False. 
  
  def apt_install(packages=[], force=False, warn_only=False): 
  Confirm or install packages using apt. Takes a single string 
  with a package name or a list of package names. Exposed as a task.
  
  src_install(pkg_url, configure_opts='', 
                     make_cmd='make', make_opts='',
                     install_cmd='make install', install_opts='', 
                     warn_only=False): Given the URL of a 
  source tarball, unpacks it in /tmp, compiles, and installs. 
  Note that there is no attempt to deal with prerequisites or 
  post-download patching; for those write a tarball-specific 
  function. Exposed as a task.
  
  capture_install(package, use='python', use_pip=False,
                       use_sudo=False, user=None): Use pip or 
  easy_install to install the named package (optionally using a 
  specific Python binary). Made to work with virtualenvs, but 
  works outside, too. Captures the location of the installed 
  package(s) as well as the location and names of any binaries. 
  Returns a dict describing the install information. Requires 
  that distribute or pip be installed in whatever Python 
  environment is being used.
  
  The capture_install function is not exposed as a task, but it
  is the backend behind both pip_install and easy_install.
  
  pip_install(package, use='python', use_sudo=False, user=None):
  Use pip to install the named package into the Python 
  environment defined by the provided Python executable. Works 
  with virtualenvs. Puts installed executables from the package, 
  if any, into either the virtualenv or global bin directory.
  Exposed as a task.
  
  easy_install(package, use='python', use_sudo=False, user=None):
  Use distribute/easy_install to install the named package 
  into the Python environment defined by the provided Python 
  executable. Works with virtualenvs. Puts installed executables 
  from the package, if any, into either the virtualenv or global 
  bin directory. Exposed as a task.
  
"""
# stdlib imports
import re

# Third party imports
from fabric.api import *
import fabric.contrib.files as remote
from fabric.contrib.console import confirm

# Package imports
from .. import constants
from ..utils import test_if, which, srun, exists, is_dir, report
from ..utils import mktempdir, download_remote

__all__ = ['update_apt', 'update_packages', 'add_apt_repository',
           'check_installed', 'apt_install', 'src_install', 
           'capture_install', 'pip_install', 'easy_install']

@task
@runs_once
def update_apt():
  """
  Run apt-get update.
  """
  report('Updating apt package lists... ')
  with settings(warn_only=True):
    output = sudo('apt-get -qq update')
  report('Done.\n')


@task
def update_packages():
  """
  Runs apt-get upgrade to pick up security fixes and updates.
  """
  update_apt()
  report('Updating installed packages... ')
  output = sudo('apt-get -qq -y upgrade')
  report('Done.\n')


@task
def add_apt_repository(repo):
  """
  Runs add-apt-repository to add the requested repo.
  """
  report('Adding repository {}...'.format(repo))
  sudo('add-apt-repository -y {}'.format(repo))
  report('Updating apt package lists... ')
  with settings(warn_only=True):
    output = sudo('apt-get -qq update')
  report('Done.\n')


def _normalize_pkglist_input(package_name_or_list):
  """
  Given either a package name (as a string) or a list of names
  (as a list, tuple, or other iterable), return a list of package
  names.
  """
  packages = package_name_or_list
  if hasattr(packages, 'translate'):
    # Packages looks like a string.
    packages = [packages]
  # Force evaluation into a list
  return list(packages)


def check_installed(package):
  """
  Use a variety of methods to check whether a package is installed.
  Uses which, dpkg, and locate (if available) to find the requested 
  package. This is a heuristic and can be fooled, so be aware.
  Returns True or False.
  """
  package = package.lower()
  # Try which
  result = which(package)
  if result: 
    report('which {} => {}'.format(package, result))
    return True
  # Try dpkg
  with hide('everything'):
    inst = [l.strip() for l in srun('dpkg --list').split('\n')]
  for line in inst:
    if not line.startswith('ii'): continue
    ii, name, version, description = line.split(None, 3)
    if package in name.lower(): 
      report('dpkg --list includes:\n{}'.format(line))
      return True
  # Try locate
  locate_cmd = 'locate {}'.format(package)
  locate_available = test_if(srun, locate_cmd)
  if locate_available:
    with hide('everything'):
      found = [l.strip() for l in srun(locate_cmd).split('\n')]
    for line in found:
      last_element = line.split('/')[-1]
      if (last_element.strip() and 
          last_element.lower().startswith(package)):
        report('locate {} includes:\n{}'.format(package, line))
        return True
  return False


@task
def apt_install(packages=[], force=False, warn_only=False):
  """
  Confirm or install packages using apt. Takes a single string with 
  a package name, or a list of package names.
  """
  update_apt()
  pkglist = _normalize_pkglist_input(packages)
  pkgstring = ' '.join(pkglist)
  if force:
    force_cmd = ' -f '
  else:
    force_cmd = ''
  apt_install_cmd = 'apt-get -q {} -y install {}'
  if warn_only:
    # Return True (success)
    with settings(warn_only=True):
      output = srun(apt_install_cmd.format(force_cmd,pkgstring),
                    use_sudo=True)
      return output.return_code
  else:
    srun(apt_install_cmd.format(force_cmd,pkgstring), use_sudo=True)


@task
def src_install(pkg_url, configure_opts='', 
                  make_cmd='make', make_opts='',
                  install_cmd='make install', install_opts='', 
                  warn_only=False):
  """
  Given the URL of a source tarball, unpacks it in /tmp, compiles,
  and installs. Note that there is no attempt to deal with
  prerequisites or post-download patching; for those write a 
  tarball-specific function.
  """
  src_pkg = pkg_url.split('/')[-1]
  tempdir = mktempdir()
  src_dir = './'
  with cd(tempdir):
    download_remote(pkg_url)
    tgz_exts = ['.tgz', '.tar.gz']
    if any([src_pkg.endswith(ext) for ext in tgz_exts]):
      srun('tar -xzf {}'.format(src_pkg))
      for ext in tgz_exts: 
        src = src_pkg.replace(ext, '')
        if exists(src) and is_dir(src):
          src_dir = src
    if pkg_url.endswith('zip'):
      srun('unzip {}'.format(src_pkg))
      src = src_pkg.replace('.zip', '')
      if exists(src) and is_dir(src):
        src_dir = src
    bz2_exts = ['.tbz', '.tar.bz', 'tar.bz2']
    if any([src_pkg.endswith(ext) for ext in bz2_exts]):
      srun('tar -xjf {}'.format(src_pkg))
      for ext in bz2_exts: 
        src = src_pkg.replace(ext, '')
        if exists(src) and is_dir(src):
          src_dir = src
    with cd(src_dir):
      if exists('configure'):
        run('./configure {}'.format(configure_opts))
      if make_cmd:
        run('{} {}'.format(make_cmd, make_opts))
      if install_cmd:
        sudo('{} {}'.format(install_cmd, install_opts))
  srun('rm -rf {}'.format(tempdir), use_sudo=True)


_re_setuptools_install = r"Installing (?P<bin_name>\S+) script "\
                       r"to (?P<bin_loc>\S+)"
_re_eggpath = r"Using (?P<egg_loc>\S+)"
_re_pip_install = r"changing mode of (?P<bin_loc>(?!build)\S+)"\
                r"[/\\](?P<bin_name>\S+) to 755"
_re_alt_pip_install = r"Installing (?P<bin_name2>\S+) script to "\
                    r"(?P<bin_loc2>\S+)"


def _update_match(match, prev=None, collapse=None):
  """
  Loop through a series of regular expression matches updating a 
  dict of results. Return a dict where the values are provided in 
  the regular expression (using Python's (?P<name>) format) and 
  the values are the matched parts of the string. If there is more 
  than one value found, create a list with all found values.
  
  The collapse argument takes a series of matchname pairs and 
  treats them equivalently for purposes of updating the match. For 
  example, the collapse argument ('bin_name', 'bin_name2') would 
  find matches that were named either 'bin_name' or 'bin_name2' and 
  places them both into the match info dict at 'bin_name' (the 
  first argument).
  """
  if prev is None: prev = {}
  if match is None: return prev
  if collapse:
    collapsedict = {}
    for wanted, second in collapse:
      collapsedict[second] = wanted
  matchinfo = match.groupdict()
  for m in matchinfo:
    key, val = m, matchinfo[m]
    if not val: continue
    if collapse:
      if key in collapsedict: key = collapsedict[key]
    report("Found install value for %s: %s" % (key, val))
    if key not in prev:
      prev[key] = set([val])
    else:
      if hasattr(prev[key], 'union'):
        prev[key].add(val)
      else:
        prev[key] = set([prev[key], val])
  for key in prev:
    if len(prev[key]) == 1:
      prev[key] = list(prev[key])[0]
  return prev


def capture_install(package, python='', use_pip=False,
                       use_sudo=False, user=None):
  """
  Use pip or easy_install to install the named package (optionally 
  using a specific Python binary). Made to work with virtualenvs, 
  but works outside, too. Captures the location of the installed 
  package(s) as well as the location and names of any binaries. 
  Returns a dict describing the install information.
  
  Requires that distribute or pip be installed in whatever Python 
  environment is being used.
  """
  if not python:
    if 'installed_python' in env and env.installed_python:
      python = env.installed_python
    else: 
      python = pypath('python', version=USE_PYTHON_VERSION)
  package_list = _normalize_pkglist_input(package)
  if use_pip:
    argv = ['pip', 'install'] + package_list
    install = "import sys;sys.argv = %s;"\
            "from pkg_resources import load_entry_point;"\
            "sys.exit(load_entry_point("\
            "'pip>=1.0','console_scripts','pip')())" % repr(argv)
    _REcapture = '|'.join([_re_pip_install, _re_alt_pip_install])
  else:
    install = "import sys;sys.argv.extend(%s); "\
              "from pkg_resources import load_entry_point; "\
              "sys.exit(load_entry_point('distribute>=0.6.14', "\
              "'console_scripts','easy_install')())" % package_list
    _REcapture = '|'.join([_re_setuptools_install, _re_eggpath])      
  cmd = '%s -c "%s"' % (python, install)
  output = srun(cmd, use_sudo=use_sudo, user=user)
  install_info = {}
  for match in re.finditer(_REcapture, output):
    install_info = _update_match(match, prev=install_info,
      collapse=(use_pip and (('bin_loc', 'bin_loc2'), 
                              ('bin_name', 'bin_name2'))))
  return install_info


@task
def pip_install(package, python='', use_sudo=False, 
                  user=None):
  """
  Use pip to install the named package into the Python 
  environment defined by the provided Python executable. Works 
  with virtualenvs. Puts installed executables from the package, 
  if any, into either the virtualenv or global bin directory.
  """
  info = capture_install(package, python, True, use_sudo, user)


@task
def easy_install(package, python='', use_sudo=False, 
                   user=None):
  """
  Use distribute/easy_install to install the named package 
  into the Python environment defined by the provided Python 
  executable. Works with virtualenvs. Puts installed executables 
  from the package, if any, into either the virtualenv or global 
  bin directory.
  """
  info = capture_install(package, python, False, use_sudo, user)



#~