Commits

Vladimir Mihailenco committed a658fff Merge

Merge with tip

Comments (0)

Files changed (19)

 recursive-include fab_deploy_tests .gitignore README
 
 # this won't take an effect unless  --no-prune is used:
-#     ./setup sdist --no-prune
+#     ./setup.py sdist --no-prune
 recursive-include fab_deploy_tests/test_project2/.git *
 CHANGES
 =======
 
+0.6.1 (2011-03-16)
+------------------
+
+- ``verify_exists`` argument of ``fab_deploy.utils.upload_config_template``
+  function was renamed to ``skip_unexistent``;
+- ``fab_deploy.utils.upload_config_template`` now passes all extra
+  kwargs directly to fabric's ``upload_template`` (thanks Vladimir Mihailenco);
+- ``fab_deploy.virtualenv.pip_setup_conf`` command for uploading pip.conf
+  (thanks Vladimir Mihailenco);
+- ``fab_deploy.deploy.push`` no longer calls 'synccompress' management command;
+- ``fab_deploy.deploy.push`` accepts 'before_restart' keyword argument -
+  that's a callable that will be executed just before code reload;
+- fixed regression in ``fab_deploy.deploy.push`` command: 'notest' argument
+  was incorrectly renamed to 'test';
+- customization docs are added.
+
 0.6 (2011-03-11)
 ----------------
 - custom project layouts support (thanks Vladimir Mihailenco):
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6'
+release = '0.6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/custom_layouts.rst

-Custom project layouts
-======================
-
-:doc:`guide` describes standard project layout::
-
-    my_project
-        ...
-        config_templates <- this folder should be copied from django-fab-deploy
-            apache.config
-            django_wsgi.py
-            hgrc
-            nginx.config
-
-        reqs             <- a folder with project's pip requirement files
-            all.txt      <- main requirements file, list all requirements in this file
-            active.txt   <- put recently modified requirements here
-            ...          <- you can provide extra files and include them with '-r' syntax in e.g. all.txt
-
-        fabfile.py       <- your project's Fabric deployment script
-        config.py        <- this file should be included in settings.py and ignored in .hgignore
-        config.server.py <- this is a production django config template
-        settings.py
-        manage.py
-
-django-fab-deploy does not enforce this layout. Requirements handling,
-config templates placement, local settings file names and project source
-folder can be customized using these options:
-
-* :attr:`env.conf.PROJECT_PATH`
-* :attr:`env.conf.LOCAL_CONFIG`
-* :attr:`env.conf.REMOTE_CONFIG_TEMPLATE`
-* :attr:`env.conf.CONFIG_TEMPLATES_PATHS`
-* :attr:`env.conf.PIP_REQUIREMENTS_PATH`
-* :attr:`env.conf.PIP_REQUIREMENTS`
-* :attr:`env.conf.PIP_REQUIREMENTS_ACTIVE`
-
-Example
--------
-
-Let's configure django-fab-deploy to use the following layout::
-
-    my_project
-        hosting                 <- a folder with server configs
-            staging             <- custom configs for 'staging' server
-                apache.config   <- custom apache config for staging server
-
-            production          <- custom configs for 'production' server
-                apache.config
-                nginx.config
-
-            apache.config       <- default configs
-            django_wsgi.py
-            nginx.config
-
-        src                     <- django project source files
-            apps
-                ...
-
-            local_settings.py   <- local settings
-            stage_settings.py   <- local settings for staging server
-            prod_settings.py    <- local settings for production server
-
-            settings.py
-            manage.py
-
-        requirements.txt        <- single file with all pip requirements
-        fabfile.py              <- project's Fabric deployment script
-
-It uses subfolder for storing django project sources, single pip requirements
-file and different config templates for different servers in
-non-default locations.
-
-fabfile.py::
-
-    from fab_deploy import *
-
-    # common layout options
-    COMMON_OPTIONS = dict(
-        PROJECT_PATH = 'src',
-        LOCAL_CONFIG = 'local_settings.py',
-        PIP_REQUIREMENTS = 'requirements.txt',
-        PIP_REQUIREMENTS_ACTIVE = 'requirements.txt',
-        PIP_REQUIREMENTS_PATH = '',
-    )
-
-    def staging():
-        env.hosts = ['user@staging.example.com']
-        env.conf = COMMON_OPTIONS.copy()
-        env.conf.update(
-            REMOTE_CONFIG_TEMPLATE = 'stage_settings.py',
-            CONFIG_TEMPLATES_PATHS = ['hosting/staging', 'hosting'],
-        )
-        update_env()
-
-    def production():
-        env.hosts = ['user@example.com']
-        env.conf = COMMON_OPTIONS.copy()
-        env.conf.update(
-            REMOTE_CONFIG_TEMPLATE = 'prod_settings.py',
-            CONFIG_TEMPLATES_PATHS = ['hosting/production', 'hosting'],
-        )
-        update_env()
-

docs/customization.rst

+Customization
+=============
+
+.. _custom-deployment-scripts:
+
+Custom deployment scripts
+-------------------------
+
+django-fab-deploy is intended to be a library, not a framework.
+So the preferred way for customizing standard command is to just
+wrap it or to create a new command by combining existing commands::
+
+    # fabfile.py
+    from fab_deploy import *
+    import fab_deploy.deploy
+
+    @run_as('root')
+    def install_java():
+        run('aptitude update')
+        run('aptitude install -y default-jre')
+
+    def full_deploy():
+        install_java()
+        fab_deploy.deploy.full_deploy()
+
+
+:func:`fab_deploy.deploy.push` accepts callable 'before_restart'
+keyword argument. This callable will be executed after code uploading
+but before the web server reloads the code.
+
+.. _fab-push-customization:
+
+An example of 'fab push' customization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # fabfile.py
+    from fab_deploy import *
+    import fab_deploy.deploy
+
+    @inside_src
+    def rebuild_docs():
+        with cd('docs'):
+            run ('rm -rf ./_build')
+            run('make html > /dev/null')
+
+    def push(*args):
+
+        # run local tests before pushing
+        local('./runtests.sh')
+
+        # rebuild static files before restarting the web server
+        def before_restart():
+            manage('collectstatic --noinput')
+            manage('assets rebuild')
+
+        # execute default push command
+        fab_deploy.deploy.push(*args, before_restart=before_restart)
+
+        # rebuild developer documentation after pushing
+        rebuild_docs()
+
+
+.. _custom-project-layouts:
+
+Custom project layouts
+----------------------
+
+:doc:`guide` describes standard project layout::
+
+    my_project
+        ...
+        config_templates <- this folder should be copied from django-fab-deploy
+            ...
+
+        reqs             <- a folder with project's pip requirement files
+            all.txt      <- main requirements file, list all requirements in this file
+            active.txt   <- put recently modified requirements here
+            ...          <- you can provide extra files and include them with '-r' syntax in e.g. all.txt
+
+        config.py        <- this file should be included in settings.py and ignored in .hgignore
+        config.server.py <- this is a production django config template
+        fabfile.py       <- your project's Fabric deployment script
+        settings.py
+        manage.py
+
+django-fab-deploy does not enforce this layout. Requirements handling,
+config templates placement, local settings file names and project source
+folder can be customized using these options:
+
+* :attr:`env.conf.PROJECT_PATH`
+* :attr:`env.conf.LOCAL_CONFIG`
+* :attr:`env.conf.REMOTE_CONFIG_TEMPLATE`
+* :attr:`env.conf.CONFIG_TEMPLATES_PATHS`
+* :attr:`env.conf.PIP_REQUIREMENTS_PATH`
+* :attr:`env.conf.PIP_REQUIREMENTS`
+* :attr:`env.conf.PIP_REQUIREMENTS_ACTIVE`
+
+Example
+~~~~~~~
+
+Let's configure django-fab-deploy to use the following layout::
+
+    my_project
+        hosting                 <- a folder with server configs
+            staging             <- custom configs for 'staging' server
+                apache.config   <- custom apache config for staging server
+
+            production          <- custom configs for 'production' server
+                apache.config
+                nginx.config
+
+            apache.config       <- default configs
+            django_wsgi.py
+            nginx.config
+
+        src                     <- django project source files
+            apps
+                ...
+
+            local_settings.py   <- local settings
+            stage_settings.py   <- local settings for staging server
+            prod_settings.py    <- local settings for production server
+
+            settings.py
+            manage.py
+
+        requirements.txt        <- single file with all pip requirements
+        fabfile.py              <- project's Fabric deployment script
+
+It uses subfolder for storing django project sources, single pip requirements
+file and different config templates for different servers in
+non-default locations.
+
+fabfile.py::
+
+    from fab_deploy import *
+
+    # Common layout options.
+    # They are separated in this example in order to stay DRY.
+    COMMON_OPTIONS = dict(
+        PROJECT_PATH = 'src',
+        LOCAL_CONFIG = 'local_settings.py',
+        PIP_REQUIREMENTS = 'requirements.txt',
+        PIP_REQUIREMENTS_ACTIVE = 'requirements.txt',
+        PIP_REQUIREMENTS_PATH = '',
+    )
+
+    def staging():
+        env.hosts = ['user@staging.example.com']
+        env.conf = COMMON_OPTIONS.copy()
+        env.conf.update(
+            REMOTE_CONFIG_TEMPLATE = 'stage_settings.py',
+            CONFIG_TEMPLATES_PATHS = ['hosting/staging', 'hosting'],
+        )
+        update_env()
+
+    def production():
+        env.hosts = ['user@example.com']
+        env.conf = COMMON_OPTIONS.copy()
+        env.conf.update(
+            REMOTE_CONFIG_TEMPLATE = 'prod_settings.py',
+            CONFIG_TEMPLATES_PATHS = ['hosting/production', 'hosting'],
+        )
+        update_env()
+
 
 .. autofunction:: fab_deploy.utils.inside_project
 
+.. autofunction:: fab_deploy.utils.inside_src
+
 .. autofunction:: fab_deploy.utils.run_as
             active.txt   <- put recently modified requirements here
             ...          <- you can provide extra files and include them with '-r' syntax in e.g. all.txt
 
-        fabfile.py       <- your project's Fabric deployment script
         config.py        <- this file should be included in settings.py and ignored in .hgignore
         config.server.py <- this is a production django config template
+        fabfile.py       <- your project's Fabric deployment script
         settings.py
         manage.py
 
 .. note::
 
     django-fab-deploy does not enforce this layout; if it doesn't fit for some
-    reason, take a look at :doc:`custom_layouts`.
+    reason, take a look at :ref:`custom-project-layouts`.
 
 The project is now ready to be deployed.
 
        fab up:my_branch
 
 Full list of commands can be found :doc:`here <reference>`.
+
+:doc:`Customization guide <customization>` is also worth reading.
    :maxdepth: 2
 
    guide
-   custom_layouts
+   customization
    fabfile
    reference
    testing

fab_deploy/deploy.py

     virtualenv.virtualenv_create()
     make_clone()
 
-    virtualenv.pip_setup_conf()
     virtualenv.pip_install(env.conf.PIP_REQUIREMENTS, restart=False)
 
     setup_web_server()
     apache.apache_setup()
     nginx.nginx_setup()
 
-def push(*args):
+def push(*args, **kwargs):
     ''' Run it instead of your VCS push command.
 
-    Arguments:
+    The following strings are allowed as positional arguments:
 
-    * test - don't run dj_cmd.tests
-    * syncdb - run dj_cmd.syncdb before code reloading
-    * migrate - run dj_cmd.migrate before code reloading
-    * pip_update - run virtualenv.pip_update before code reloading
-    * norestart - do not reload source code
+    * 'notest' - don't run tests
+    * 'syncdb' - run syncdb before code reloading
+    * 'migrate' - run migrate before code reloading
+    * 'pip_update' - run virtualenv.pip_update before code reloading
+    * 'norestart' - do not reload source code
+
+    Keyword arguments:
+
+    * before_restart - callable to be executed after code uploading
+      but before the web server reloads the code.
+
+    Customization example can be found  :ref:`here <fab-push-customization>`.
+
     '''
-    allowed_args = set(['test', 'syncdb', 'migrate', 'pip_update', 'norestart'])
+    allowed_args = set(['notest', 'syncdb', 'migrate', 'pip_update', 'norestart'])
     for arg in args:
         if arg not in allowed_args:
             puts('Invalid argument: %s' % arg)
         dj_cmd.syncdb()
     if 'migrate' in args:
         dj_cmd.migrate()
-    dj_cmd.compress()
+
+    # execute 'before_restart' callback
+    kwargs.get('before_restart', lambda: None)()
+
     if 'norestart' not in args:
         apache.touch()
-    if 'test' not in args:
+    if 'notest' not in args:
         dj_cmd.test()
 
 def undeploy(confirm=True):

fab_deploy/system.py

     os = utils.detect_os()
     if os in ['lenny', 'squeeze'] and env.conf.SUDO_USER == 'root':
         install_sudo()
-    
+
     setup_backports()
     install_common_software()
 
         utils.safe_sudo('aptitude update')
 
 def create_linux_account(pub_key_file):
-    """ Creates linux account and setups ssh access. """
+    """ Creates linux account, setups ssh access and pip.conf file. """
     with open(os.path.normpath(pub_key_file), 'rt') as f:
         ssh_key = f.read()
-    username = env.conf['USER']
+
+    username = env.conf.USER
+
+    @utils.run_as(username)
+    def setup_pip_conf():
+        from fab_deploy import virtualenv
+        virtualenv.pip_setup_conf()
+
     with (settings(warn_only=True)):
         utils.safe_sudo('adduser %s --disabled-password --gecos ""' % username)
-        with cd('/home/' + username):
+        with cd(env.conf.HOME_DIR):
             utils.safe_sudo('mkdir -p .ssh')
             files.append('.ssh/authorized_keys', ssh_key, use_sudo=True,
                          user=env.conf.SUDO_USER)
             utils.safe_sudo('chown -R %s:%s .ssh' % (username, username))
+        setup_pip_conf()
+
 
 def ssh_add_key(pub_key_file):
     """ Adds a ssh key from passed file to user's authorized_keys on server. """

fab_deploy/utils.py

 __all__ = ['run_as', 'update_env', 'inside_project', 'inside_virtualenv',
            'delete_pyc', 'print_env', 'detect_os', 'safe_sudo']
 
+
 def _codename(distname, version, id):
     patterns = [
         ('lenny', ('debian', '^5', '')),
         return inner
     return decorator
 
-def upload_config_template(name, to=None, verify_exists=False, **kwargs):
+def upload_config_template(name, to=None, skip_unexistent=False, **kwargs):
     if to is None:
         base_dir = env.conf['ENV_DIR'] + "/etc/"
         run('mkdir -p ' + base_dir)
         to = base_dir + name
     config_template = _config_template_path(name)
-    if verify_exists and (not config_template
-                          or not os.path.exists(config_template)):
+
+    if config_template is None and skip_unexistent:
         return
-    files.upload_template(config_template, to, context=env.conf, use_jinja=True,
+    files.upload_template(config_template, to, env.conf, use_jinja=True,
                           **kwargs)
 
 def update_env():
     return inner
 
 def inside_src(func):
+    """
+    Decorator. Use it to perform actions inside remote source dir
+    (repository root) with virtualenv activated.
+    """
     @wraps(func)
     def inner(*args, **kwargs):
         with cd(env.conf.SRC_DIR):
 
 def inside_project(func):
     """
-    Decorator. Use it to perform actions inside project dir with
+    Decorator. Use it to perform actions inside remote project dir
+    (that's a folder where :file:`manage.py` resides) with
     virtualenv activated::
 
         from fabric.api import *

fab_deploy/vcs/hg.py

     local('hg push ssh://%s/src/%s/' % (env.hosts[0], env.conf.INSTANCE_NAME))
 
 def configure():
-    upload_config_template('hgrc', env.conf['SRC_DIR'] + '/.hg/hgrc')
+    upload_config_template('hgrc', env.conf.SRC_DIR + '/.hg/hgrc',
+                           skip_unexistent=True)

fab_deploy/virtualenv.py

 def pip_setup_conf():
     """ Sets up pip.conf file """
     utils.upload_config_template('pip.conf',
-        env.conf.HOME_DIR + '/.pip/pip.conf', verify_exists=True)
+        env.conf.HOME_DIR + '/.pip/pip.conf', skip_unexistent=True)
 
 def virtualenv_create():
     run('mkdir -p envs')

fab_deploy_tests/runtests.py

 
     FabDeployTest.vm_name = sys.argv[1]
     common_tests = load([BasicTest, SshTest, MysqlTest, CrontabTest])
+    fast_tests = load([FastPrepareServerTest, ApacheSetupTest,
+                       PipSetupTest, NoPipSetupTest])
     suites = {
-        'fast': TestSuite(common_tests + load([FastPrepareServerTest, ApacheSetupTest])),
+        'fast': TestSuite(common_tests + fast_tests),
         'slow': TestSuite(load([DeployTest, CustomLayoutDeployTest])),
         'prepare': TestSuite(common_tests + load([PrepareServerTest])),
     }

fab_deploy_tests/test_project2/hosting/pip.conf

+[global]
+timeout = 60

fab_deploy_tests/tests/__init__.py

 from .utils_tests import *
 from .crontab import *
 from .deploy import *
+from .virtualenv import *

fab_deploy_tests/tests/deploy.py

 from __future__ import absolute_import
 import os
-import urllib2
 from fab_deploy.utils import run_as
 from fabric.api import *
 from fabtest import fab, vbox_urlopen
-from fab_deploy.deploy import deploy_project, push, undeploy, full_deploy
+from fab_deploy.deploy import deploy_project, push, undeploy
 from fab_deploy.mysql import mysql_create_db
 from fab_deploy.apache import (apache_make_config, apache_make_wsgi,
                                apache_restart, apache_install)
     def assertResponse(self, url, data):
         self.assertEqual(vbox_urlopen(url).read(), data)
 
+    def assertNoFile(self, path):
+        with(settings(warn_only=True)):
+            res = get_file_content(path)
+        self.assertFalse(res.succeeded)
+
+    def assertFileExists(self, path):
+        with(settings(warn_only=True)):
+            self.assertTrue(get_file_content(path))
+
     def setup_conf(self):
         self.cwd = os.getcwd()
         os.chdir(self.project)
 
         foo_port = env.conf.APACHE_PORT
         self.assertPortNotBinded(foo_port)
-        self.assertTrue(get_file_content('/etc/apache2/sites-enabled/foo'))
+        self.assertFileExists('/etc/apache2/sites-enabled/foo')
 
         fab(apache_restart)
         self.assertPortBinded(foo_port)
         self.assertPortBinded(bar_port)
 
     def test_apache_make_wsgi(self):
+        self.assertNoFile(env.conf.ENV_DIR+'/var/wsgi/foo.py')
         fab(apache_make_wsgi)
-        wsgi_file = get_file_content(env.conf.ENV_DIR+'/var/wsgi/foo.py')
-        self.assertTrue(wsgi_file)
+        self.assertFileExists(env.conf.ENV_DIR+'/var/wsgi/foo.py')
 
 
 class DeployTest(FabDeployProjectTest):
         # deploying project again should be possible
         fab(deploy_project)
         self.assertResponse(url, 'foo')
+
+    def test_push_callback(self):
+        url = 'http://foo.example.com/instance/'
+
+        def before_restart():
+            before_restart.called = True
+        before_restart.called = False
+
+        def my_push(*args):
+            return push(*args, before_restart=before_restart)
+
+        fab(foo_site2)
+        fab(deploy_project)
+        fab(my_push)
+        self.assertTrue(before_restart.called)

fab_deploy_tests/tests/virtualenv.py

+from __future__ import absolute_import
+from fabric.api import *
+from fabtest import fab
+from fab_deploy.virtualenv import pip_setup_conf
+from .deploy import FabDeployProjectTest
+
+class NoPipSetupTest(FabDeployProjectTest):
+    def test_no_pip_conf(self):
+        self.assertNoFile(env.conf.HOME_DIR+'/pip.conf')
+        fab(pip_setup_conf)
+        self.assertNoFile(env.conf.HOME_DIR+'/pip.conf')
+
+class PipSetupTest(FabDeployProjectTest):
+    project = 'test_project2'
+
+    def test_pip_conf(self):
+        self.assertNoFile(env.conf.HOME_DIR+'/pip.conf')
+        fab(pip_setup_conf)
+        self.assertFileExists(env.conf.HOME_DIR+'/pip.conf')
     if cmd in sys.argv:
         from setuptools import setup
 
-version='0.6'
+version='0.6.1'
 
 setup(
     name='django-fab-deploy',