Commits

Colin Copeland committed 612afaa

update deployment

  • Participants
  • Parent commits 71b3aed

Comments (0)

Files changed (7)

 
 Ubuntu packages::
 
-    aptitude install memcached python-setuptools rsync screen git-core bzr mercurial subversion python-imaging build-essential python-dev postfix nginx byobu postgresql python-psycopg2
+    aptitude install memcached python-setuptools rsync screen git-core bzr mercurial subversion python-imaging build-essential python-dev postfix nginx byobu postgresql python-psycopg2 python-gevent
 
 Python packages::
 
 import os
+import random
+import string
+from fabric.api import cd, env, local, require, run, sudo
+from fabric.contrib.files import exists, upload_template
 
-from fabric.api import *
-from fabric.contrib.project import rsync_project
-from fabric.contrib import files, console
-from fabric import utils
-from fabric.decorators import hosts
 
-
-env.home = '/home/copelco/'
 env.project = 'copelco'
+env.user = 'website'
+env.home = '/home/%(user)s/' % env
+env.repo = u'https://copelco@bitbucket.org/copelco/copelco'
 # remove -l from env.shell, "mesg n" in ~/.profile was causing issues
 # see Why do I sometimes see ``err: stdin: is not a tty``?
 # http://github.com/bitprophet/fabric/blob/master/FAQ
 
 
 def _setup_path():
-    env.repo_root = os.path.join(env.root, env.environment)
-    env.code_root = os.path.join(env.root, env.environment, 'lib', env.project)
-    env.virtualenv_root = os.path.join(env.root, env.environment, 'env')
+    env.root = os.path.join(env.home, 'www', env.environment)
+    env.log_dir = os.path.join(env.root, 'log')
+    env.code_root = os.path.join(env.root, 'code_root')
+    env.project_root = os.path.join(env.code_root, 'lib', env.project)
+    env.virtualenv_root = os.path.join(env.root, 'env')
+    env.services = os.path.join(env.home, 'services')
     env.db = '%s_%s' % (env.project, env.environment)
-    env.settings = '%(project)s.settings_%(environment)s' % env
+    env.vhost = '%s_%s' % (env.project, env.environment)
 
 
 def staging():
     """ run commands on the remote staging environment """
-    env.user = 'copelco'
     env.environment = 'staging'
-    env.hosts = ['50.17.226.149']
-    env.root = os.path.join(env.home, 'www')
-    env.repo = 'https://bitbucket.org/copelco/copelco'
+    env.hosts = ['ec2-50-19-40-110.compute-1.amazonaws.com']
     env.branch = 'default'
+    env.server_port = '7000'
     _setup_path()
 
 
 def production():
     """ run commands on the remote staging environment """
-    env.user = 'copelco'
     env.environment = 'production'
-    env.hosts = ['50.17.226.149']
-    env.root = os.path.join(env.home, 'www')
-    env.repo = 'https://bitbucket.org/copelco/copelco'
+    env.hosts = ['ec2-50-19-40-110.compute-1.amazonaws.com']
     env.branch = 'default'
+    env.server_port = '8000'
     _setup_path()
 
 
     """ create virtual environment on remote host """
     require('virtualenv_root', provided_by=('production', 'staging'))
     args = '--clear --distribute'
-    run('rm -rf %s' % env.virtualenv_root)
     run('virtualenv -q %s %s' % (args, env.virtualenv_root))
 
 
-def update_requirements(run_migrate=True):
+def update_requirements():
     """ update remote Python environment """
     require('code_root', provided_by=('production', 'staging'))
-    req_dir = os.path.join(env.root, env.environment, 'requirements')
-    with cd(req_dir):
-        for file_name in ['apps.txt']:
-            cmd = ['pip install']
-            cmd += ['-q -E %(virtualenv_root)s' % env]
-            cmd += ['--requirement %s' % file_name]
-            run(' '.join(cmd))
-    if run_migrate:
-        migrate()
+    requirements = os.path.join(env.code_root, 'requirements')
+    with cd(requirements):
+        cmd = ['pip install']
+        cmd += ['-q -E %(virtualenv_root)s' % env]
+        cmd += ['--requirement %s' % os.path.join(requirements, 'apps.txt')]
+        run(' '.join(cmd))
 
 
 def clone():
     """ clone github repository on remote machine """
-    require('root', provided_by=('production', 'staging'))
+    require('code_root', provided_by=('production', 'staging'))
+    run('mkdir -p %(root)s' % env)
     with cd(env.root):
-        run('hg clone -q %s %s' % (env.repo, env.environment))
-    with cd(os.path.join(env.root, env.environment)):
-        run('hg branch %s' % env.branch)
+        run('hg clone %(repo)s %(code_root)s' % env)
 
 
-def pull(run_migrate=True):
+def update_source():
     """ pull latest code to remote environment """
     require('repo_root', provided_by=('production', 'staging'))
-    with cd(env.repo_root):
-        run('hg pull -u')
-    migrate()
+    with cd(env.code_root):
+        run('hg pull')
+        run('hg update %(branch)s' % env)
 
 
-def deploy():
-    require('repo_root', provided_by=('production', 'staging'))
-    pull()
-    supervisor_restart()
+def upload_supervisor_conf():
+    """ upload Supervisor configuration from the template """
+    require('environment', provided_by=('staging', 'production'))
+    template = os.path.join(os.path.dirname(__file__), 'services', 'templates', 'supervisor.conf')
+    destination = os.path.join(env.services, 'supervisor', '%(environment)s.conf' % env)
+    upload_template(template, destination, context=env)
+    _supervisor_command('update')
 
-def run_tests():
-    """ run tests locally """
-    local('./manage.py test', capture=False)
 
+def upload_nginx_conf():
+    """ upload Nginx configuration from the template """
+    require('environment', provided_by=('staging', 'production'))
+    template = os.path.join(os.path.dirname(__file__), 'services', 'templates', 'nginx.conf')
+    destination = os.path.join(env.services, 'nginx', '%(environment)s.conf' % env)
+    upload_template(template, destination, context=env)
+    restart_nginx()
 
-def touch():
-    """ reload remote wsgi process """
-    require('code_root', provided_by=('production', 'staging'))
-    with cd(env.code_root):
-        run('touch %s.wsgi' % env.environment)
 
-
-def update_apache_conf():
-    """ upload apache configuration """
-    source = os.path.join(env.project, 'apache', '%(environment)s.conf' % env)
-    dest = os.path.join(env.home, 'apache.conf.d')
-    put(source, dest, mode=0755)
-    _toggle_monit(enabled=False)
-    apache_reload()
-    _toggle_monit(enabled=True)
+def upload_gunicorn_conf():
+    """ upload Gunicorn configuration from the template """
+    require('environment', provided_by=('staging', 'production'))
+    template = os.path.join(os.path.dirname(__file__), 'services', 'templates', 'gunicorn.conf')
+    destination = os.path.join(env.services, 'gunicorn', '%(environment)s.py' % env)
+    upload_template(template, destination, context=env)
 
 
 def update_services():
-    """ upload changes to services such as nginx """
-    rsync_project(remote_dir=env.home, local_dir="services")
-    nginx_reload()
-    supervisor_restart()
-    #netstat_plnt()
+    """ upload changes to services configurations as nginx """
+    upload_supervisor_conf()
+    upload_nginx_conf()
+    upload_gunicorn_conf()
+    netstat_plnt()
 
 
-def kick():
-    """ restart nginx and apache on a remote host """
-    _toggle_monit(enabled=False)
-    nginx_restart()
-    apache_restart()
-    _toggle_monit(enabled=True)
-    netstat_plnt()
-
 def netstat_plnt():
     """ run netstat -plnt on a remote host """
     require('hosts', provided_by=('production', 'staging'))
     run('sudo netstat -plnt')
 
 
-def configtest():
-    """ run configtest on remote machine """
-    require('hosts', provided_by=('production', 'staging'))
-    run('apache2ctl configtest')
-
-
-def nginx_reload():
-    """ reload nginx on remote machine """
-    require('hosts', provided_by=('production', 'staging'))
-    run('sudo /etc/init.d/nginx reload')
-
-
-def nginx_restart():
+def restart_nginx():
     """ restart nginx on remote machine """
     require('hosts', provided_by=('production', 'staging'))
     run('sudo /etc/init.d/nginx restart')
 
 
-def apache_reload():
-    """ reload apache on remote machine """
-    require('hosts', provided_by=('production', 'staging'))
-    run('sudo /etc/init.d/apache2 reload')
+def _supervisor_command(command):
+    sudo('supervisorctl %s' % command)
 
 
-def apache_restart():    
-    """ restart apache on remote machine """
-    require('hosts', provided_by=('production', 'staging'))
-    run('sudo /etc/init.d/apache2 restart')
+def restart_server():
+    """ restart gunicorn server """
+    require('environment', provided_by=('staging', 'production'))
+    _supervisor_command('restart %(environment)s:%(environment)s-server' % env)
 
 
-def supervisor_restart():
-    """ restart apache on remote machine """
-    require('hosts', provided_by=('production', 'staging'))
-    run('sudo /etc/init.d/supervisor stop')
-    run('sudo /etc/init.d/supervisor start')
+def restart_supervisor():
+    """ restart all Supervisor controlled processes """
+    require('environment', provided_by=('staging', 'production'))
+    _supervisor_command('restart %(environment)s:*' % env)
 
-def mkdirs():
-    """ create directory structure on remote mahcine """
-    run('mkdir -p %(root)s' % env)
-    logs = os.path.join(env.home, 'www', 'log')
-    run('mkdir -p %s' % logs)
-    run('chmod -R a+w %s' % logs)
+
+def restart_all():
+    """ restart Nginx and Supervisor controlled processes """
+    restart_nginx()
+    restart_supervisor()
 
 
 def bootstrap():
     """ bootstrap remote environment """
     require('hosts', provided_by=('production', 'staging'))
-    mkdirs()
     clone()
     create_virtualenv()
-    update_requirements(run_migrate=False)
+    update_requirements()
     setup_dir_permissions()
 
-def _toggle_monit(enabled):
-    require('hosts', provided_by=('production', 'staging'))
-    if enabled:
-        run('sudo service monit start')
-    else:
-        run('sudo service monit stop')
-    
-
-def enable_upgrade_message():    
-    """ enable upgrade message on remote environment """
-    require('code_root', provided_by=('production', 'staging'))
-    _toggle_monit(enabled=False)
-    _toggle_upgrade(enabled=True)
-
-
-def disable_upgrade_message():
-    """ disable upgrade message on remote environment """
-    require('code_root', provided_by=('production', 'staging'))
-    _toggle_monit(enabled=True)
-    _toggle_upgrade(enabled=False)
-
-
-def _toggle_upgrade(enabled):
-    wsgi_file = os.path.join(env.code_root,
-                             '%s.wsgi' % env.environment)
-    if enabled:
-        before = "SHOW_UPGRADE_MESSAGE = False"
-        after = "SHOW_UPGRADE_MESSAGE = True"
-    else:
-        before = "SHOW_UPGRADE_MESSAGE = True"
-        after = "SHOW_UPGRADE_MESSAGE = False"
-    files.sed(wsgi_file, before, after)
-    touch()
-
 
 def setup_dir_permissions():
     """ make upload media dirs writable """
-    require('code_root', provided_by=('production', 'staging'))
-    writable_media = (
-        'uploads',
-        'cms_page_media',
-        'protected',
-        'images',
-        'avatars',
-        'img/classifieds',
-        'photologue/photos',
-    )
-    media_root = os.path.join(env.code_root, 'media')
-    for directory in writable_media:
-        run('chmod -R a+w %s' % os.path.join(media_root, directory))
-    run('chmod -R a+w %s' % os.path.join(env.code_root, 'sf_index'))
+    require('environment', provided_by=('production', 'staging'))
+    run('mkdir -p %(log_dir)s' % env)
+    #sudo('chown :www-data %(log_dir)s' % env, user='copelco')
+    #sudo('chmod g+w %(log_dir)s' % env, user='copelco')
+    run('mkdir -p %(services)s/nginx' % env)
+    run('mkdir -p %(services)s/supervisor' % env)
+    run('mkdir -p %(services)s/gunicorn' % env)
 
 
-def migrate():
-    """ run south migration on remote environment """
-    require('code_root', provided_by=('production', 'staging'))
-    settings = 'copelco.local_settings'
-    migrate = './manage.py migrate --settings=%s' % settings
-    activate = 'source %s' % os.path.join(env.virtualenv_root, 'bin/activate')
-    with cd(env.code_root):
-        run('%s && %s' % (activate, migrate))
+def syncdb():
+    """ run syncdb and south migrations """
+    require('environment', provided_by=('staging', 'production'))
+    run('%(virtualenv_root)s/bin/python %(project_root)s/manage.py syncdb --noinput' % env)
+    run('%(virtualenv_root)s/bin/python %(project_root)s/manage.py migrate --noinput' % env)
 
 
-def full_update():
-    """ install latest code and re-create virtualenv """
-    require('code_root', provided_by=('production', 'staging'))
-    pull()
-    create_virtualenv()
+def deploy():
+    """ deploy to a given environment """
+    require('environment', provided_by=('staging', 'production'))
+    update_source()
     update_requirements()
+    syncdb()
+    collectstatic()
+    restart_server()
 

File lib/copelco/manage.py

     try:
         import copelco.settings # Assumed to be in the same directory.
     except ImportError:
+        raise
         import sys
         sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
         sys.exit(1)

File lib/copelco/settings.py

     'storages',
     'south',
     'gunicorn',
+    'djcelery',
     'copelco.apps.default',
     'copelco.apps.estate',
 ]
 )
 
 LOGIN_REDIRECT_URL = '/'
+
+import djcelery
+djcelery.setup_loader()

File requirements/apps.txt

 django==1.3
-gunicorn==0.12
+gunicorn==0.12.1
 south==0.7.3
 geopy==0.94
 googlemaps==1.0.2
 django-imagekit==0.3.5
 django-storages==1.1.1
 BeautifulSoup==3.2.0
+celery==2.2.6
+django-celery==2.2.4

File services/templates/nginx.conf

     access_log %(log_dir)s/access.log;
     error_log %(log_dir)s/error.log;
 
-    location /media {
-        root %(code_root)s;
+    location /media/ {
+        alias /home/copelco/www/production/uploads/;
     }
 
     location /static {

File services/templates/supervisor.conf

 [program:%(environment)s-server]
-process_name=%%(program_name)s
 command=%(virtualenv_root)s/bin/gunicorn_django -c %(services)s/gunicorn/%(environment)s.py local_settings.py
 directory=%(project_root)s
 user=%(user)s