Commits

Brent Tubbs committed 3c82061

major fabfile cleanup

Comments (0)

Files changed (4)

 setup(
     name='silk-deployment',
     author='Brent Tubbs',
-    version='0.0.8',
+    version='0.0.9',
 	packages=find_packages(),
     package_dir={'silk': 'silk'},
     include_package_data=True,
         ],
     },
 	install_requires = [
-        'gunicorn',
-        'cherrypy',
+        'gunicorn',#for running in production
+        'cherrypy',#for 'silk run' (local devserver)
         'Fabric >= 0.9.2',
         'setproctitle',
         'PyYAML',
  - 'silk', An alias for Fabric's 'fab'.  (This is largely to minimize confusion
  at work, where we already have a different internal project called 'fab'.)
  - 'freeze2yaml', a tool to which you can pipe the output of 'pip freeze', and
- get back a yaml-formatted list of python packages.
+ get back a yaml-formatted list of python packages for storing in silk's deps.yaml
+ format.
+ - 'yaml2freeze', does the reverse of freeze2yaml.
 
 The file/directory structure of a silk-enabled project looks like this:
 
 ├── deps.yaml
 ├── fabfile.py
 ├── hello_django
-│   ├── __init__.py
-│   ├── manage.py
-│   ├── settings.py
-│   ├── static
-│   │   └── test.txt
-│   └── urls.py
 ├── membrane.py
 ├── roles
 │   ├── dev.yaml
  CherryPy, may be made available in the future.)
  - Silk embraces Fabric instead of eschewing it.
  - Silk does not yet provide any persistence (database) support, other than 
- helping you inject environment variables into your app at runtime.
+ helping you inject role-specific configuration into your app at runtime.
 
 FUTURE PLANS
 
-#ubuntu packages required by silk
+#Thi file lists packages that need to be installed on the remote
+#host in order for Silk to do its thing.
 apt_packages:
   - python-virtualenv
   - mercurial
   - git-core
   - nginx
   - supervisor
-#install build dependencies for these packages
 apt_build_deps:
-#Python packages, generated by 'pip freeze | freeze2yaml'
 python_packages:
-- Fabric==0.9.2
 - PyYAML==3.09
 - gunicorn==0.11.1
-- paramiko==1.7.6
-- pycrypto==2.0.1
 - setproctitle==1.1
 import yaml
 import random
 
+SRV_ROOT = '/srv'
+ROLLBACK_CAP = 3
+
 def _get_silk_deps():
     silk_deps_file = pkg_resources.resource_filename('silk', 'deps.yaml')
     return yaml.safe_load(open(silk_deps_file).read())
     return yaml.safe_load(open(site_deps_file).read())
 
 def _set_vars():
-    """Loads deployment settings and returns them in a dict"""
+    """
+    Loads deployment settings into Fabric's global 'env' dict
+    """
     env.local_root = silk.lib.get_site_root(os.getcwd())
     env.config = silk.lib.get_site_config(env.local_root)
-    env.silk_root='/srv'
     if len(env.roles) == 0:
         sys.exit("ERROR: you must define a role with -R <rolename>")
     elif len(env.roles) > 1:
     else:
         env.config.update(silk.lib.get_role_config(env.roles[0]))
     env.site = env.config['site']
-    env.site_root = os.path.join(env.silk_root, env.site)
-    env.rollback_cap = 5
-    env.srvdir = os.path.join(env.silk_root, env.site)
-    env.envdir = os.path.join(env.srvdir, 'env')
-    env.workdir = os.path.join('/tmp', env.site)
-    env.rollbackdir = os.path.join(env.silk_root, 'rollbacks')
+    env.remote_root = '/'.join([SRV_ROOT, env.site])
+    env.envdir = '/'.join([env.remote_root, 'env'])
+    env.workdir = '/'.join(['/tmp', env.site])
+    env.rollbackdir = '/'.join([SRV_ROOT, 'rollbacks'])
     env.deploytime = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
-    #set default socket to bind to
+    #default to using a unix socket for nginx->gunicorn
     if 'bind' not in env.config['gunicorn']:
         env.config['gunicorn']['bind'] = 'unix:/tmp/%s.sock' % env.config['site']
     env.config['bind'] = env.config['gunicorn']['bind']
 
 _set_vars()
 
+# UGLY MAGIC
+# Here we're (ab)using Fabric's built in 'role' feature to work with the way 
+# we're loading context-specific config.  Could possibly use a refactor to 
+# avoid Fabric roles altogether.
 def _get_hosts():
     """Return list of hosts to push to"""
     return env.config['push_hosts']
 
 for role in silk.lib.get_role_list(env.local_root):
     env.roledefs[role] = _get_hosts
+# END UGLY MAGIC
 
 def _put_dir(local_dir, remote_dir):
-    """Copy a local directory to a remote one, using tar and put. Silently
+    """
+    Copies a local directory to a remote one, using tar and put. Silently
     overwrites remote directory if it exists, creates it if it does not
-    exist."""
+    exist.
+    """
     local_tgz = "/tmp/fabtemp.tgz"
     remote_tgz = os.path.basename(local_dir) + ".tgz"
     local('tar -C "{0}" -czf "{1}" .'.format(local_dir, local_tgz))
         .format(remote_dir, remote_tgz))
 
 def _get_blame():
-    """Writes a yaml file that contains the site config, plus some deployment info"""
+    """
+    Returns a yaml file that contains the site config, plus some deployment info
+    """
     blame = [
         {'deployed_by': env.user,
         'deployed_from': os.uname()[1],
     return yaml.safe_dump(blame, default_flow_style=False)
 
 def _write_blame():
+    """
+    Writes blame file on remote host.
+    """
     #randomize the string to reduce chance of collision
     blamefile = '/tmp/%s-blame.yaml' % ''.join([random.choice('abcdefghijklmnopqrstuvwxyz') for x in xrange(5)])
     f = open(blamefile, 'w')
     os.remove(blamefile)
 
 def reload():
+    """
+    Reloads supervisord and nginx configs.
+    """
     print "RELOADING CONFIGS"
     sudo('supervisorctl reload')
     sudo('/etc/init.d/nginx reload')
 
 def restart():
+    """
+    Restarts nginx and supervisord.  Normally not needed (reload() is enough)
+    """
     print "RESTARTING SERVICES"
     sudo('/etc/init.d/supervisor stop; /etc/init.d/supervisor start')
     sudo('/etc/init.d/nginx restart')
 
 def archive():
+    """
+    Creates rollback archive of already-deployed site.  Rotates old rollback files.
+    """
     print "CREATING ROLLBACK"
     if not exists(env.rollbackdir, use_sudo=True):
         sudo('mkdir -p %s' % env.rollbackdir)
         sudo('rm %s' % oldest_rollback) 
 
     #then increment the numbers on the existing rollbacks
-    for i in xrange(env.rollback_cap - 1, 0, -1):
+    for i in xrange(ROLLBACK_CAP - 1, 0, -1):
         rollback_file = '%s/%s-rollback_%s.tar.bz2' % (env.rollbackdir, env.site, i)
         if exists(rollback_file):
             newname = '%s/%s-rollback_%s.tar.bz2' % (env.rollbackdir, env.site, i + 1)
             sudo('mv %s %s' % (rollback_file, newname))
 
-    #now archive env.srvdir if it exists
-    if exists(env.srvdir):
+    #now archive env.remote_root if it exists
+    if exists(env.remote_root):
         sudo('tar -cjf %(rollbackdir)s/%(site)s-rollback_1.tar.bz2 --exclude "*.log" -C %(silk_root)s %(site)s' % env)
 
 def rollback():
+    """
+    Untars most recent rollback archive and sets it running.
+    """
     print "ROLLING BACK"
     rollback_file = '%s/%s-rollback_1.tar.bz2' % (env.rollbackdir, env.site)
     if exists(rollback_file): 
 
         #move current code into oldddir
         olddir = os.path.join('/tmp', 'old-%s' % env.site)
-        if exists(env.srvdir, use_sudo=True):
-            sudo('mv %s %s' % (env.srvdir, olddir))
+        if exists(env.remote_root, use_sudo=True):
+            sudo('mv %s %s' % (env.remote_root, olddir))
 
         #move new code into srvdir
-        sudo('mv %s/%s %s' % (tmpdir, env.site, env.srvdir))
+        sudo('mv %s/%s %s' % (tmpdir, env.site, env.remote_root))
 
         #remove olddir
         if exists(olddir, use_sudo=True):
         sudo('rm -rf %s' % rollback_file)
 
         #decrement the other rollback files
-        for i in xrange(2, env.rollback_cap + 1, 1):
+        for i in xrange(2, ROLLBACK_CAP + 1, 1):
             oldname = '%s/%s-rollback_%s.tar.bz2' % (env.rollbackdir, env.site, i)
             newname = '%s/%s-rollback_%s.tar.bz2' % (env.rollbackdir, env.site, i - 1)
             if exists(oldname):
 
 
 def install_apt_deps():
+    """
+    Installs system packages and build dependencies with apt.
+    """
     for deps_dict in (env.config['silk_deps'], env.config['site_deps']):
         print deps_dict
         if deps_dict['apt_build_deps']:
 
 #TODO: rebuild virtualenv if it exists but the python version is wrong
 def create_virtualenv():
-    """Make virtualenv"""
+    """
+    Creates a virtualenv for the site.  Automatically builds egenix-mx-tools in it, since
+    pip doesn't seem able to install that.
+    """
     print "CREATING VIRTUALENV"
-    if not exists(env.srvdir, use_sudo=True):
-        sudo('mkdir -p %s' % env.srvdir)
+    if not exists(env.remote_root, use_sudo=True):
+        sudo('mkdir -p %s' % env.remote_root)
     sudo('virtualenv --no-site-packages --python=%s %s' % (env.config['runtime'], env.envdir))
     build_mx_tools()
 
 def install_python_deps():
+    """
+    Runs pip install inside the remote virtualenv.  First for silk's dependencies and
+    then for the site's.
+    """
     for dep_dict in (env.config['silk_deps'], env.config['site_deps']):
         if dep_dict['python_packages']:
             sudo('%s/bin/pip install %s' % (
             ))
 
 def install_deps():
+    """
+    Wraps the apt deps, virtualenv creation, and python deps functions to ensure
+    that things are done in the right order.
+    """
     print "INSTALLING DEPENDENCIES"
     install_apt_deps()
     if not exists(env.envdir, use_sudo=True):
     install_python_deps()
 
 def build_mx_tools():
-    """Build and install egenix-mx-tools into virtualenv"""
+    """
+    Builds and install egenix-mx-tools into virtualenv
+    """
     #egenix-mx-tools includes the mxdatetime module, which is
     #a psycopg2 dependency.  Unfortunately it's not packaged in 
     #a way that pip can install.  So we build it here instead
     sudo('rm -rf %s' % build_dir)
 
 def push_code():    
-    """Pushes site to remote host"""
+    """
+    Pushes site to remote host
+    """
     print "PUSHING CODE TO HOST"
     if exists(env.workdir):
         sudo('rm %s -rf' % env.workdir)
       return ''
 
 def write_config():
+    """
+    Creates and upload config files for nginx, supervisord, and blame.yaml
+    """
     print "WRITING CONFIG"
     nginx_static = ''
     static_dirs = env.config.get('static_dirs', None)
       for item in static_dirs:
           nginx_static += _get_nginx_static_snippet(
               item['url_path'],
-              os.path.join(env.site_root, item['system_path'])
+              '/'.join([env.remote_root, item['system_path']])
           )
     template_vars = {
         'cmd': silk.lib.get_gunicorn_cmd(env.config, bin_dir='%s/bin' % (env.envdir)),
     }
     template_vars.update(env)
     template_vars.update(env.config)
-    config_dir = os.path.join(env.workdir, 'conf')
+    config_dir = '/'.join([env.workdir, 'conf'])
     #make sure the conf and logs dirs are created
     _ensure_dir(config_dir)
-    _ensure_dir(os.path.join(env.workdir, 'logs'))
+    _ensure_dir('/'.join([env.workdir, 'logs']))
 
     template_list = (
-        ('supervisord.conf',os.path.join(config_dir, 'supervisord.conf')),
-        ('nginx.conf',os.path.join(config_dir, 'nginx.conf')),
+        ('supervisord.conf','/'.join([config_dir, 'supervisord.conf'])),
+        ('nginx.conf','/'.join([config_dir, 'nginx.conf'])),
     )
     _write_templates(template_list, template_vars)
     _write_blame()
 
 def switch():
+    """
+    Does a little dance to move the old project dir out of the way and put
+    the new one in its place.
+    """
     print "MOVING NEW CODE INTO PLACE"
     #copy the virtualenv into env.workdir
     sudo('cp %s %s/ -r' % (env.envdir, env.workdir))
-    #move old env.srvdir
-    olddir = os.path.join('/tmp', 'old-%s' % env.site)
-    sudo('mv %s %s' % (env.srvdir, olddir))
+    #move old env.remote_root
+    olddir = '/'.join(['/tmp', 'old-%s' % env.site])
+    sudo('mv %s %s' % (env.remote_root, olddir))
     #move code into place
-    sudo('mv %s %s' % (env.workdir, env.srvdir))
+    sudo('mv %s %s' % (env.workdir, env.remote_root))
 
 def cleanup():
+    """
+    Removes the old project dir.  (But you still have a rollback!)
+    """
     print "CLEANING UP"
     #do this last to minimize time between taking down old site and setting up new one
     #since "mv" is faster than "cp -r" or "rm -rf"
-    olddir = os.path.join('/tmp', 'old-%s' % env.site)
+    olddir = '/'.join(['/tmp', 'old-%s' % env.site])
     sudo('rm %s -rf' % olddir)
 
 def server_setup():
-    """Sets up nginx and supervisord global config files"""
+    """
+    Installs nginx and supervisord on remote host.
+    Sets up nginx and supervisord global config files.
+    """
     install_apt_deps()
     template_list = (
         ('supervisord_root.conf', '/etc/supervisor/supervisord.conf'),
     restart()
 
 def push():
+    """
+    The main function.  Assuming you have nginx and supervisord installed
+    and configured, this function will put your site on the remote host and
+    get it running.
+    """
     archive()
     install_deps()
     push_code()
     cleanup()
 
 def dump_config():
-    """Just loads up the config and prints it out.
-    For testing."""
+    """
+    Just loads up the config and prints it out. For testing.
+    """
     print env.config
-
-def blah():
-    """dummy function for testing things"""
-    print _get_blame()
-
-def hostname():
-    """Used for testing"""
-    print os.uname()[1]