christchurch_django2 / fabfile.py

from collections import namedtuple
from datetime import datetime
from fabric.api import run, local, abort, env, put
from fabric.contrib import files
from fabric.contrib import console
from fabric.decorators import hosts, runs_once
from fabric.context_managers import cd, lcd, settings, hide
import os
import os.path
join = os.path.join
import sys

#  fabfile for deploying Christ Church website
#
# == Overview ==
#
# === Development ===
#
# You need a root directory to hold everything, and the following
# sub directories:
#
#  project/    - holds a checkout of this repository
#                i.e. fabfile.py and siblings live in that dir.
#
#  usermedia/  - corresponds to MEDIA_ROOT
#
# === Deployment ===
#
# There are two targets, STAGING and PRODUCTION, which live on the same
# server. They are almost identical, with these differences:
# - STAGING is on staging.christchurchbradford.org.uk
# - PRODUCTION is on www.christchurchbradford.org.uk
# - They have different databases
# - They have different apps on the webfaction server
#    - for the django project app
#    - for the static app
#
# settings_priv.py and settings.py controls these things.
#
# In each target, we aim for atomic switching from one version to the next.
# This is not quite possible, but as much as possible the different versions
# are kept separate, preparing the new one completely before switching to it.
#
# To achieve this, new code is uploaded to a new 'dest_dir' which is timestamped,
# inside the 'src' dir in the christchurch app directory.

# /home/cciw/webapps/christchurch/         # PRODUCTION or
# /home/cciw/webapps/christchurch_staging/ # STAGING
#    src/
#       src-2010-10-11_07-20-34/
#          env/                    # virtualenv dir
#          project/                # uploaded from local
#          static/                 # built once uploaded
#       current/                   # symlink to src-???

# At the same level as 'src-2010-10-11_07-20-34', there is a 'current' symlink
# which points to the most recent one. The apache instance looks at this (and
# the virtualenv dir inside it) to run the app.

# There is a webfaction app that points to src/current/static for serving static
# media. (One for production, one for staging). There is also a 'christchurch_usermedia'
# app which is currently shared between production and staging. (This will only
# be a problem if usermedia needs to be re-organised).

# Code is deployed by pushing dev code to bitbucket, then pulling.

# When deploying, once the new directory is ready, the apache instance is
# stopped, the database is upgraded, and the 'current' symlink is switched. Then
# the apache instance is started.

# The information about this layout is unfortunately spread around a couple of
# places - this file and the settings file - because it is needed in both at
# different times.

# === Bootstrapping ===
#
# Steps needed on a new server:
# - create Django app on webfaction.
# - create static app on webfaction
# - create usermedia app on webfaction
# - create database on webfaction
#
# # Make layout and modify various things:
# $ cd ~/webapps/christchurch_django/
# $ mkdir src
# $ mkdir db
# $ mv myproject.wsgi project.wsgi
# $ replace myproject christchurch -- project.wsgi
# $ rm -rf myproject/ lib/
# $ emacs apache/conf/http.conf
# - Remove python path
# - Change myproject -> project
# $ emacs apache/bin/start
# - Add:
#   . $HOME/webapps/christchurch_django/src/current/env/bin/activate

# Then deploy using this fabfile

# Then create webfaction symlink app for static media

# Same again for christchurch_django_staging, with a different
# webapp dir and venv dir.

# Remember to add password info to ~/.pgpass

env.hosts = ["cciw@christchurchbradford.org.uk"]

this_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(this_dir)
webapps_root = '/home/cciw/webapps'

# The path (relative to parent_dir) to where the project source code is stored:
project_dir = 'project'
usermedia_local = os.path.join(parent_dir, 'usermedia')
usermedia_production = os.path.join(webapps_root, 'christchurch_usermedia')

class Target(object):
    """
    Represents a place where the project is deployed to.

    """
    def __init__(self, django_app='', dbname=''):
        self.django_app = django_app
        self.dbname = dbname

        self.webapp_root = join(webapps_root, self.django_app)
        # src_root - the root of all sources on this target.
        self.src_root = join(self.webapp_root, 'src')
        self.current_version = SrcVersion('current', join(self.src_root, 'current'))

    def make_version(self, label):
        return SrcVersion(label, join(self.src_root, "src-%s" % label))

class SrcVersion(object):
    """
    Represents a version of the project sources on a Target
    """
    def __init__(self, label, src_dir):
        self.label = label
        # src_dir - the root of all sources for this version
        self.src_dir = src_dir
        # venv_dir - note that _update_virtualenv assumes this relative layout
        # of the 'env' dir and the 'project' dir.
        self.venv_dir = join(self.src_dir, 'env')
        # project_dir - where the CHRISTCHURCH project srcs are stored.
        self.project_dir = join(self.src_dir, project_dir)
        # static_dir - this is defined this way in settings.py
        self.static_dir = join(self.src_dir, 'static')

        self.additional_sys_paths = [project_dir]

STAGING = Target(
    django_app = "christchurch_django_staging",
    dbname = "cciw_christchurch_staging",
)

PRODUCTION = Target(
    django_app = "christchurch_django",
    dbname = "cciw_christchurch",
)

def backup_database(target, version):
    fname = "%s-%s.db" % (target.dbname, version.label)
    fullname = join(target.webapp_root, 'db', fname)
    run("dump_pg_db.sh %s %s" % (target.dbname, fullname))


def run_venv(command, **kwargs):
    run("source %s/bin/activate" % env.venv + " && " + command, **kwargs)


def virtualenv(venv_dir):
    """
    Context manager that establishes a virtualenv to use,
    """
    return settings(venv=venv_dir)


def _update_symlink(target, version):
    if files.exists(target.current_version.src_dir):
        run("rm %s" % target.current_version.src_dir) # assumes symlink
    run("ln -s %s %s" % (version.src_dir, target.current_version.src_dir))


def _update_virtualenv(version):
    # Update virtualenv in new dir.
    with cd(version.src_dir):
        # We should already have a virtualenv, but it will need paths updating
        run("virtualenv --python=python2.7 env")
        # Need this to stop ~/lib/ dirs getting in:
        run("touch env/lib/python2.7/sitecustomize.py")
        with virtualenv(version.venv_dir):
            with cd(version.project_dir):
                run_venv("pip install -r requirements.txt")

        # Need to add project to path.
        # Could do 'python setup.py develop' but not all projects support it
        pth_file = '\n'.join("../../../../" + n for n in version.additional_sys_paths)
        pth_name = "deps.pth"
        with open(pth_name, "w") as fd:
            fd.write(pth_file)
        put(pth_name, join(version.venv_dir, "lib/python2.7/site-packages"))
        os.unlink(pth_name)


def _stop_apache(target):
    run(join(target.webapp_root, "apache2/bin/stop"))


def _start_apache(target):
    run(join(target.webapp_root, "apache2/bin/start"))


def _restart_apache(target):
    with settings(warn_only=True):
        _stop_apache(target)
    _start_apache(target)

def _update_project_sources(target, version):
    # This also copies the virtualenv which is contained in the same folder,
    # which saves a lot of time with installing.

    run("mkdir -p %s" % version.src_dir)
    with cd(version.src_dir):
        if files.exists(target.current_version.project_dir + "/.hg"):
            # Clone local copy if we can
            run("hg clone %s project" % target.current_version.project_dir)
        else:
            run("hg clone ssh://hg@bitbucket.org/spookylukey/christchurch_django project")

        with cd(version.project_dir):
            # We update to the version that is currently checked out locally,
            # because, at least for staging, it might not be the tip of default.
            current_rev = local("hg id -i", capture=True)
            run("hg pull ssh://hg@bitbucket.org/spookylukey/christchurch_django")
            run("hg update -r %s" % current_rev.strip("+"))

        # Avoid recreating the virtualenv if we can
        if files.exists(target.current_version.venv_dir):
            run("cp -a -L %s %s" % (target.current_version.venv_dir,
                                    version.src_dir))

    # Also need to sync files that are not in main sources VCS repo.
    local("rsync christchurch/settings_priv.py cciw@christchurchbradford.org.uk:%s/christchurch/settings_priv.py" % version.project_dir)

def _build_static(version):
    # This always copies all files anyway, and we want to delete any unwanted
    # files, so we start from clean dir.
    run("rm -rf %s" % version.static_dir)

    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            # django-cms requires appmedia, which overlaps a lot with Django's
            # own 'statcifiles', but with different conventions (appmedia looks
            # for 'media' directories and copies to MEDIA_ROOT, Django looks for
            # 'static' and copies to STATIC_ROOT).
            run_venv("./manage.py collectstatic -v 0 --noinput")
            run_venv("./manage.py symlinkmedia")

    run("chmod -R ugo+r %s" % version.static_dir)


def _update_db(target, version):
    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            run_venv("./manage.py syncdb")
            run_venv("./manage.py migrate --all")


def _deploy(target):
    label = datetime.now().strftime("%Y-%m-%d_%H.%M.%S")
    version = target.make_version(label)

    _update_project_sources(target, version)
    _update_virtualenv(version)
    _build_static(version)

    # Ideally, we:
    # 1) stop web server
    # 2) updated db
    # 3) rollback if unsuccessful.
    # 4) restart webserver

    # In practice, for this low traffic site it is better to keep website
    # going for as much time as possible, and cope with any small bugs that
    # come from mismatch of db and code.

    db_backup_name = backup_database(target, version)
    _update_db(target, version)
    _stop_apache(target)
    _update_symlink(target, version)
    _start_apache(target)


def _clean(target):
    """
    Misc clean-up tasks
    """
    # Remove old src versions.
    with cd(target.src_root):
        with hide("stdout"):
            currentlink = run("readlink current").split('/')[-1]
            otherlinks = set([x.strip() for x in run("ls src-* -1d").split("\n")])
        otherlinks.remove(currentlink)
        otherlinks = list(otherlinks)
        otherlinks.sort()
        otherlinks.reverse()

        # Leave the most recent previous version, delete the rest
        for d in otherlinks[1:]:
            run("rm -rf %s" % d)


def deploy_staging():
    _deploy(STAGING)


def deploy_production():
    _deploy(PRODUCTION)


def stop_apache_production():
    _stop_apache(PRODUCTION)


def stop_apache_staging():
    _stop_apache(STAGING)


def start_apache_production():
    _start_apache(PRODUCTION)


def start_apache_staging():
    _start_apache(STAGING)


def restart_apache_production():
    _restart_apache(PRODUCTION)


def restart_apache_staging():
    _restart_apache(STAGING)


def clean_staging():
    _clean(STAGING)


def clean_production():
    _clean(PRODUCTION)


def test_staging():
    _test_remote(STAGING)


def test_production():
    _test_remote(PRODUCTION)


def upload_usermedia():
    local("chmod ugo+r  -R %s/*" % usermedia_local)
    local("rsync -z -r --exclude='*.mp3' %s/ cciw@christchurchbradford.org.uk:%s" % (usermedia_local, usermedia_production), capture=False)


def upload_sermons():
    local("chmod ugo+r  %s/downloads/sermons/*" % usermedia_local)
    local("rsync -v --progress --size-only %s/downloads/sermons/* cciw@christchurchbradford.org.uk:%s/downloads/sermons" % (usermedia_local, usermedia_production))


def backup_usermedia():
    local("rsync -z -r  cciw@christchurchbradford.org.uk:%s/ %s" % (usermedia_production, usermedia_local), capture=False)



# TODO:
#  - backup db task. This should be run only in production, and copies
#    files to Amazon S3 service.
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.