Source

stacklet / src / stacklet / task / distro / debu.py

# Copyright (C) 2009, dfn@stacklet.com
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

"""Tasks for creating debian or ubuntu images.

Please see docstring of each class for usage.

"""
from stacklet.util.objectify import _XO_
from stacklet.util.objectify import *
from stacklet.task.stacktask import *
from stacklet.task.stackerror import *

import ConfigParser
import string
import fnmatch, re

class _XO_debootstrap(_XO_, StackTask):
    """Debootstrap a debian or ubuntu image.  Image will be chrootable upon completion
       of this task with network access to a package repository.


    Task Arguments:
    arch -- the architecture eg 'amd64'.
    codename -- what to install, eg 'quantal''
    proxy -- (optional) proxyhost[:port]

    """
    _TASK_NAME = 'debootstrap'

    def __init__(self):
        pass

    def process(self, buildContext):
        try:
            try:
                self._buildContext = buildContext
                self.log_info(buildContext, 'entering')
                self.precheck(buildContext)


                env = {'http_proxy': self._proxy, 'PATH': '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'}

                cmd = 'debootstrap --arch=%s --verbose --keep-debootstrap-dir %s %s' % (self._arch, self._codename, buildContext.mountPoint)
                self.exec_command(buildContext, cmd, env=env)


            except TaskError, taskErr:

                taskErr.criticality = TaskError.ABORT_BUILD
                raise
        finally:
            self.log_info(buildContext, 'leaving')

    def precheck(self, buildContext):
        self.is_valid_build_context(buildContext)

        self._arch = self.get_value('arch')
        self._codename = self.get_value('codename')
        self._proxy = self.get_value('proxy')

        if self.is_empty(self._arch):
            raise FieldError(self._TASK_NAME, TaskError.ABORT_TASK, 'arch is required')

        if self.is_empty(self._codename):
            raise FieldError(self._TASK_NAME, TaskError.ABORT_TASK, 'codename is required')


class _XO_bootstrap(_XO_, StackTask):
    """Bootstrap a debian or ubuntu image.  Image will be chrootable upon completion
       of this task with network access to a package repository.


    Task Arguments:
    url -- the location to download packages from.
    proxy -- (optional) proxyhost[:port]

    """
    _TASK_NAME = 'bootstrap'

    def __init__(self):
        pass

    def process(self, buildContext):
        try:
            try:
                self._buildContext = buildContext
                self.log_info(buildContext, 'entering')
                self.precheck(buildContext)

                config = ConfigParser.RawConfigParser()
                config.read(self._bootstrapConfig)

                env = {'http_proxy': self._proxy, 'PATH': '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'}

                cmd = 'cp /etc/resolv.conf ' +  buildContext.mountPoint + '/etc/'
                self.exec_command(buildContext, cmd)

                cmd = 'wget -4 -c -nv --base=' + self._url + ' -P ' + buildContext.mountPoint + '/bootstrap -i ' + self._corePackages
                self.exec_command(buildContext, cmd, env=env)

                p = re.compile( '(\n|\r)')

                stage1debs = p.sub('', config.get('stage1', 'packages')).split(',')
                stage2debs = p.sub('', config.get('stage2', 'packages')).split(',')
                stage3debs = p.sub('', config.get('stage3', 'packages')).split(',')

                self.extract(stage1debs, env)
                self.install(stage2debs, env, '--force-all')
                self.install(stage3debs, env, '')

            except TaskError, taskErr:

                taskErr.criticality = TaskError.ABORT_BUILD
                raise
        finally:
            self.log_info(buildContext, 'leaving')

    def precheck(self, buildContext):
        self.is_valid_build_context(buildContext)

        self._url = self.get_value('url')
        self._proxy = self.get_value('proxy')
        self._corePackages = buildContext.mountPoint + '/bootstrap/corePackages'
        self._bootstrapConfig = buildContext.get_asset_dir() + 'bootstrapConfig'

        if self.is_empty(self._url):
            raise FieldError(self._TASK_NAME, TaskError.ABORT_TASK, 'url is required')

        if not os.access(self._corePackages, os.F_OK):
            raise BuildError(self._TASK_NAME, TaskError.ABORT_TASK, 'Could not find ' + self._corePackages)

        if not os.access(self._bootstrapConfig, os.F_OK):
            raise BuildError(self._TASK_NAME, TaskError.ABORT_TASK, 'Could not find ' + self._bootstrapConfig)

    def extract(self, debs, env):
        debs = fnmatch.filter(os.listdir(self._buildContext.mountPoint + '/bootstrap/'), "*.deb")
        for deb in debs:
            cmd = 'ar -p ' + self._buildContext.mountPoint + '/bootstrap/' + deb + ' data.tar.gz | zcat | tar -C ' + self._buildContext.mountPoint +  ' -xf - '
            self.exec_command(self._buildContext, cmd, env=env)

    def install(self, debs, env, options):
        for deb in debs:
            cmd = 'cd /bootstrap/ && dpkg  -i ' + options + ' ' + deb
            self.exec_command_chroot(self._buildContext, cmd, env=env)

            cmd = 'rm -f ' +  self._buildContext.mountPoint + '/var/run/init.upgraded'
            self.exec_command(self._buildContext, cmd)

class _XO_package(_XO_, StackTask):
    """Install and/or remove package(s)

    Task Arguments:
    add -- space-delimited list of packages to install
    remove -- space-delimited list of packages to remove
    update -- space-delimited list of packages to update
    conf -- (optional) the location of yum.conf within the image
    proxy -- (optional) proxyhost[:port]

    """
    _TASK_NAME = 'package'

    def __init__(self):
        pass

    def process(self, buildContext):
        try:
            try:
                self._buildContext = buildContext
                self.log_info(buildContext, 'entering')
                self.precheck(buildContext)
                cmds = list()

                env = {'http_proxy': self._proxy, 'PATH': '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin', 'DEBIAN_FRONTEND': 'noninteractive'}

                if not self.is_empty(self._update):
                    self.log_info(buildContext, "update " + self._update)
                    if self._update == 'all':
                        cmd = 'apt-get --yes update; apt-get --yes upgrade'
                    cmds.append(cmd)

                if not self.is_empty(self._add):
                    self.log_info(buildContext, "add " + self._add)
                    cmd = 'apt-get %s --yes install %s' % (self._options, self._add)
                    cmds.append(cmd)

                if not self.is_empty(self._remove):
                    self.log_info(buildContext, "remove " + self._remove)
                    cmd = 'apt-get --yes remove ' + self._remove
                    cmds.append(cmd)

                self.exec_command_chroot(buildContext, cmds, env=env)

            except TaskError, taskErr:
                if taskErr.criticality == TaskError.ABORT_BUILD:
                    raise
                self.log_error(buildContext, taskErr)
        finally:
            self.log_info(buildContext, 'leaving')

    def precheck(self, buildContext):
        self.is_valid_build_context(buildContext)

        self._add = self.get_value('add')
        self._remove = self.get_value('remove')
        self._update = self.get_value('update')
        self._conf = self.get_value('conf')
        self._proxy = self.get_value('proxy')
        self._options = self.get_value('options')

        if self.is_empty(self._add) and self.is_empty(self._remove) and self.is_empty(self._update):
            raise FieldError(self._TASK_NAME, TaskError.ABORT_TASK, 'add or remove or update is required')


class _XO_service(_XO_, StackTask):
    """Enable and/or disable service(s)

    Task Arguments:
    enable -- space-delimited list of services to enable
    disable -- space-delimited list of services to disable
    """
    _TASK_NAME = 'service'

    def __init__(self):
        pass

    def process(self, buildContext):
        try:
            try:
                self._buildContext = buildContext
                self.log_info(buildContext, 'entering')
                self.precheck(buildContext)
                cmds = list()

                if not self.is_empty(self._enable):
                    self.log_info(buildContext, "enable " + self._enable)
                    for serv in self._enable.split():
                        cmds.append('update-rc.d -f ' + serv +  " defaults")

                if not self.is_empty(self._disable):
                    self.log_info(buildContext, "disable " + self._disable)
                    for serv in self._disable.split():
                        cmds.append('update-rc.d -f ' + serv +  " remove")

                self.exec_command_chroot(buildContext, cmds)
            except TaskError, taskErr:
                if taskErr.criticality == TaskError.ABORT_BUILD:
                    raise
                self.log_error(buildContext, taskErr)
        finally:
            self.log_info(buildContext, 'leaving')


    def precheck(self, buildContext):
        self.is_valid_build_context(buildContext)

        self._enable = self.get_value('enable')
        self._disable = self.get_value('disable')

        if self.is_empty(self._enable) and self.is_empty(self._disable):
            raise FieldError(self._TASK_NAME, TaskError.ABORT_TASK, 'enable or disable is required')

class _XO_cleanup(_XO_, StackTask):
    """Cleanup log files, yum cache, /tmp, etc...

    Task Arguments:
    zerofs -- if 'true' the empty space in the image will be zero'ed for better compressibility
    """
    _TASK_NAME = 'cleanup'

    def __init__(self):
        pass

    def process(self, buildContext):
        try:
            try:
                self._buildContext = buildContext
                self.log_info(buildContext, 'entering')
                self.precheck(buildContext)

                cmds = list()
                cmds.append('apt-get clean')
                cmds.append('rm -f /var/log/dmesg*')
                cmds.append('truncate --size=0 /var/log/*.log')
                cmds.append('truncate --size=0 /var/log/exim4/*log')
                cmds.append('truncate --size=0 /var/log/mail.*')
                cmds.append('truncate --size=0 /var/log/messages')
                cmds.append('truncate --size=0 /var/log/syslog')
                cmds.append('truncate --size=0 /var/log/wtmp')
                cmds.append('truncate --size=0 /var/log/lastlog')
                cmds.append('truncate --size=0 /var/log/faillog')
                cmds.append('truncate --size=0 /var/log/apt/*.log')
                cmds.append('truncate --size=0 /var/log/aptitude /var/log/debug /var/log/udev /var/log/apt/*.log /var/log/ConsoleKit/history /var/run/utmp /var/log/btmp')                
                cmds.append('rm -f /var/lib/dhcp/*')
                cmds.append('rm -f /var/lib/dhcp3/*')
                cmds.append('rm -f /var/lib/dhclient/*')
                cmds.append('rm -f /var/run/*.pid')
                cmds.append('rm -rf /var/spool/mail/*')
                cmds.append('rm -rf /var/tmp/*')
                cmds.append('rm -f /var/lock/*')
                cmds.append('rm -rf /root/.ssh')
                cmds.append('rm -f /root/.bash_history')
                cmds.append('rm -f /root/.nano_history')
                cmds.append('rm -f /etc/resolv.conf')
                cmds.append('rm -rf /bootstrap')
                cmds.append('rm -rf /debootstrap')
                cmds.append('rm -f /etc/ssh/*key*')
                cmds.append('rm -rf /tmp/*')

                self.exec_command_chroot(buildContext, cmds, checkReturnValue=False)

                if self._zerofs == 'true':
                    self.zero_empty_space(buildContext)
            except TaskError, taskErr:
                if taskErr.criticality == TaskError.ABORT_BUILD:
                    raise
                self.log_error(buildContext, taskErr)
        finally:
            self.log_info(buildContext, 'leaving')

    def onerr(self, buildContext, cmd, retval):
        pass

    def precheck(self, buildContext):
        self.is_valid_build_context(buildContext)
        self._zerofs = self.get_value('zerofs')

stacklet.util.objectify._XO_bootstrap = _XO_bootstrap
stacklet.util.objectify._XO_stk_bootstrap = _XO_bootstrap
stacklet.util.objectify._XO_stk_debootstrap = _XO_debootstrap
stacklet.util.objectify._XO_package = _XO_package
stacklet.util.objectify._XO_stk_package = _XO_package
stacklet.util.objectify._XO_service = _XO_service
stacklet.util.objectify._XO_stk_service = _XO_service
stacklet.util.objectify._XO_cleanup = _XO_cleanup
stacklet.util.objectify._XO_stk_cleanup = _XO_cleanup