Commits

Brent Tubbs committed 26fceae

revamp now functional. Adding sock_http.py for better testing of whether new site is live.

Comments (0)

Files changed (6)

silk/extras/__init__.py

Empty file removed.

silk/extras/django.py

-from silk.fabfile import push as real_push
-from silk.utils import run_til_you_die as _run_til_you_die
-from signal import SIGINT
-import silk
-import os
-#SIGH
-
-#time to rewrite code that I wrote last night but is trapped on my laptop because I didn't do an hg addremove
-#SIGH
-#I think the only new file was django.py.
-#which includes collectstatic, and an overwritten push
-
-def _get_proj_dir(root):
-    """Ugly magic function to loop through subdirectories and return the first
-    one that looks like a Django project (has a settings.py)"""
-    #get list of current folder contents
-    paths = [os.path.join(root, folder) for folder in os.listdir(root) if os.path.isdir(os.path.join(os.path.join(root, folder)))]
-    for path in paths:
-        if os.path.isfile(os.path.join(path, 'settings.py')):
-            return path
-
-def collectstatic():
-    """Runs ./manage.py collectstatic, setting current Silk role 
-    as env var so it can be picked up in local_settings"""
-    args = ['./manage.py', 'collectstatic', '--settings=local_settings']
-    proj_dir = _get_proj_dir(silk.lib.get_site_root(os.getcwd()))
-    env_vars = {'SILK_ROLE': silk.lib.get_role()}
-    _run_til_you_die(args, SIGINT, proj_dir, env=env_vars)
-
-def push():
-    collectstatic()
-    real_push()

silk/extras/s3.py

-import os
-import boto
-from boto.s3.key import Key
-
-def push_s3(azn_key, azn_secret, s3bucket, localdir, s3dir):
-    """Recurses through localdir, uploading each file it finds into the corresponding
-    path in s3dir"""
-    conn = boto.connect_s3(azn_key, azn_secret)
-    bucket = conn.get_bucket(s3bucket)
-    tree = os.walk(localdir)
-    for folder in tree:
-        folderpath = folder[0]
-        subfiles = folder[2]
-        for file in subfiles:
-            localfile = os.path.join(folderpath, file)
-            trimmed_dir = folderpath[len(localdir):]
-            #os.path.normpath cleans out any double slashes //
-            #warning: if run from windows the slashes will get turned into backslashes
-            s3file = os.path.normpath("/".join((s3dir, trimmed_dir, file)))
-            print "pushing %s to %s" % (localfile, s3file)
-            k = Key(bucket)
-            k.key = s3file
-            k.set_contents_from_filename(localfile)
-            k.set_acl('public-read')
 import datetime
 import time
 import posixpath
-import tempfile
 import random
 import re
+import copy
+import yaml
 
-import yaml
 from fabric.api import *
 from fabric.colors import green, red, yellow
-from fabric.contrib.files import exists, upload_template
+from fabric.contrib.files import exists, contains, upload_template
 
 import silk.lib
 
     env.envdir = _join(env.root, 'env')
     env.rollback_cap = env.config.get('rollback_cap', DEFAULT_ROLLBACK_CAP)
 
+    # Use the default supervisord include location for the deployment's include file.
+    env.supd_conf_file = '/etc/supervisor/conf.d/%s.conf' % env.deployment
+
     # Set up gunicorn config
     default_bind = silk.lib.GUNICORN_BIND_PATTERN % env.deployment
     if 'gunicorn' in env.config:
     env.roledefs[role] = _get_hosts
 # END UGLY MAGIC
 
-def _put_dir(local_dir, remote_dir, exclude=''):
-    """
-    Copies a local directory to a remote one, using tar and put. Silently
-    overwrites remote directory if it exists, creates it if it does not
-    exist.
-    """
-    local_tarball = "/tmp/fabtemp.tar.bz2"
-    remote_tarball = os.path.basename(env.site) + ".tar.bz2"
-
-    tar_cmd = 'tar -C "%(local_dir)s" -cjf "%(local_tarball)s" %(exclude)s .' % locals()
-    local(tar_cmd)
-    put(local_tarball, remote_tarball, use_sudo=True)
-    local('rm -f "%(local_tarball)s"' % locals())
-    sudo('rm -Rf "{0}"; mkdir -p "{0}"; tar -C "{0}" -xjf "{1}" && rm -f "{1}"'\
-        .format(remote_dir, remote_tarball))
-
 def _tmpfile():
     """Generates a random filename on the remote host.  Useful for dumping
     stdout to a file that you want to download or read later.  Assumes the
     randompart = "".join([random.choice(chars) for x in xrange(20)])
     return "/tmp/silk_tmp_%s" % randompart
 
+def _put_dir(local_dir, remote_dir, exclude=''):
+    """
+    Copies a local directory to a remote one, using tar and put. Silently
+    overwrites remote directory if it exists, creates it if it does not
+    exist.
+    """
+    tarball = "%s.tar.bz2" % _tmpfile()
+
+    tar_cmd = 'tar -C "%(local_dir)s" -cjf "%(tarball)s" %(exclude)s .' % locals()
+    local(tar_cmd)
+    put(tarball, tarball, use_sudo=True)
+    local('rm -f "%(tarball)s"' % locals())
+    sudo('rm -Rf "{0}"; mkdir -p "{0}"; tar -C "{0}" -xjf "{1}" && rm -f "{1}"'\
+        .format(remote_dir, tarball))
+
 def _get_blame():
     """
-    Returns a yaml file that contains the site config, plus some deployment
-    info.  The actual blame.yaml file will be written from this data.
+    Return information about this deployment, to be written as the "blame"
+    section in the site.yaml file.
     """
-    blame = [
-        {'deployed_by': env.user,
-        'deployed_from': os.uname()[1],
-        'deployed_at': datetime.datetime.now(),
-         'deployed_role': env.roles[0]},
-        {'config':env.config}
-    ]
-    return yaml.safe_dump(blame, default_flow_style=False)
+    return {'deployed_by': env.user, 
+            'deployed_from': os.uname()[1],
+            'deployed_at': datetime.datetime.now(), 
+            'deployed_role': env.roles[0]}
 
-def _write_blame():
-    """
-    Writes blame file on remote host.
-    """
-    blamefile_name = _tmpfile()
-    blamefile = open(blamefile_name, 'w')
-    blame_txt = _get_blame()
-    blamefile.write(blame_txt)
-    blamefile.close()
-    remote_blame = _join(env.root, 'blame.yaml')
-    put(blamefile_name, remote_blame)
-    local('rm %s' % blamefile_name)
+def _write_file(path, contents, use_sudo=False, chown=None):
+    file_name = _tmpfile()
+    file = open(file_name, 'w')
+    file.write(contents)
+    file.close()
+    put(file_name, path, use_sudo=use_sudo)
+    sudo('chmod +r %s' % path)
+    if chown:
+        sudo('chown %s %s' % (chown, path))
+    local('rm %s' % file_name)
 
-    # Fix the permissions on the remote blame file
-    sudo('chmod +r %s' % remote_blame)
-
+def _write_site_yaml():
+    """Writes the site.yaml file for the deployed site."""
+    # Make a copy of env.config
+    site_yaml_dict = copy.copy(env.config)
+    # add the blame in
+    site_yaml_dict['blame'] = _get_blame()
+    # write it to the remote host
+    file = _join(env.root, 'site.yaml')
+    _write_file(file, yaml.safe_dump(site_yaml_dict, default_flow_style=False))
 
 def _write_template(template, dest, context):
     #first try to load the template from the local conf_templates dir.
       return ','.join(['%s="%s"' % (key, env_dict[key]) for key in env_dict.keys()])
     except AttributeError:
       #env_dict isn't a dictionary, so they must not have included any env vars for us.
-      #return empty string
       return ''
 
-def server_setup():
+def _green(text):
+    print green(text)
+
+def _red(text):
+    print red(text)
+
+def _yellow(text):
+    print yellow(text)
+
+def _list_dir(dirname):
+    """Given the path for a directory on the remote host, return its contents
+    as a python list."""
+    txt = sudo('ls -1 %s' % dirname)
+    return txt.split('\r\n')
+
+def _is_live(site):
+    """Returns True if site 'site' has a supervisord.conf entry, else False."""
+    # Check both the default include location and the silk <=0.2.9 location.
+    old_conf_file = _join(SRV_ROOT, site, 'conf', 'supervisord.conf')
+    return exists(env.supd_conf_file) or exists(old_conf_file)
+
+def _is_running(procname, tries=3, wait=2):
+    """Given the name of a supervisord process, tell you whether it's running
+    or not.  If status is 'starting', will wait until status has settled."""
+    # Status return from supervisorctl will look something like this::
+    # mysite_20110623_162319 RUNNING    pid 628, uptime 0:34:13
+    # mysite_20110623_231206 FATAL      Exited too quickly (process log may have details)
+
+    status_parts = sudo('supervisorctl status %s' % env.deployment).split()
+    if status_parts[1] == 'RUNNING':
+        return True
+    elif status_parts[1] == "FATAL":
+        return False
+    elif tries > 0:
+        # It's neither running nor dead yet, so try again
+        _yellow("Waiting %s seconds for process to settle" % wait)
+        time.sleep(wait)
+
+        # decrement the tries and double the wait time for the next check.
+        return _is_running(procname, tries - 1, wait * 2)
+    else:
+        return False
+
+def _is_this_site(name):
+    """Return True if 'name' matches our site name (old style Silk deployment
+    naming' or matches our name + timestamp pattern."""
+    
+    site_pattern = re.compile('%s_\d{8}_\d{6}' % env.site)
+    return (name == env.site) or (re.match(site_pattern, name) is not None)
+
+def install_server_deps():
     """
-    Installs nginx and supervisord on remote host.  Sets up nginx and
-    supervisord global config files.
+    Installs nginx and supervisord on remote Ubuntu host.
     """
-    install_apt_deps()
+    sudo('apt-get install nginx supervisor --assume-yes --quiet --no-upgrade')
 
 def push_code():
     # Push the local site to the remote root, excluding files that we don't
     # want to leave cluttering the production server
     _green("PUSHING CODE")
     exclude = ("--exclude=site.yaml --exclude=roles "
-               "--exclude=requirements.txt")
+               "--exclude=requirements.txt --exclude=fabfile.py")
     _put_dir(env.local_root, env.root, exclude)
 
 def create_virtualenv():
 
     template_vars.update(env)
     template_vars.update(env.config)
-    #make sure the conf and logs dirs are created
-    _ensure_dir(config_dir)
+    #make sure the logs dir is created
     _ensure_dir(_join(env.root, 'logs'))
 
     # Put supervisord include in default location
-    dest = '/etc/supervisor/conf.d/%s.conf' % site
-    _write_template('supervisord.conf', dest, template_vars)
+    _write_template('supervisord.conf', env.supd_conf_file, template_vars)
 
-def _green(text):
-    print green(text)
+def fix_supd_config_bug():
+    """Fixes a bug from an earlier version of Silk that wrote an invalid line
+    to the master supervisord.conf"""
+    # Silk 0.2.9 and earlier included a command to configure supervisord and
+    # nginx to include files in /srv/<site>/conf, in addition to their default
+    # include directories.  While this was valid for nginx, supervisord does
+    # not allow for multiple "files" lines in its "include" section (it does
+    # allow for multiple globs on the single "files" line, though.  This command
+    # finds the offending pattern in /etc/supervisor/supervisord.conf and
+    # replaces it with the correct equivalent.
 
-def _red(text):
-    print red(text)
+    # Note that Silk 0.3.0 and later does not require any changes from the
+    # default supervisord.conf that ships with Ubuntu.  All files are included
+    # in the supervisord's conf.d directory.
+    
+    file = '/etc/supervisor/supervisord.conf'
 
-def _yellow(text):
-    print yellow(text)
+    if contains(file, "files = /srv/\*/conf/supervisord.conf", use_sudo=True):
+        _green("FIXING OLD SUPERVISOR CONFIG BUG")
+        _yellow("See http://bits.btubbs.com/silk-deployment/issue/15/incorrect-supervisord-include-config-in")
+        
+        bad = "\r\n".join([
+            "files = /etc/supervisor/conf.d/*.conf",
+            "files = /srv/*/conf/supervisord.conf"
+        ])
 
-def _list_dir(dirname):
-    """Given the path for a directory on the remote host, return its contents
-    as a python list."""
-    cmd = 'ls -1 %s' % dirname
+        good = ("files = /etc/supervisor/conf.d/*.conf "
+                "/srv/*/conf/supervisord.conf\n")
+        
+        txt = sudo('cat %s' % file)
 
-    txt = sudo(cmd)
-
-    return txt.split('\r\n')
-
-def _is_live(site):
-    """Returns True if site 'site' has a supervisord.conf entry, else False."""
-    conf_path = '/etc/supervisor/conf.d/%s.conf' % site
-    old_conf_path = _join(SRV_ROOT, site, 'conf', 'supervisord.conf')
-    return exists(conf_path) or exists(old_conf_path)
-
-def _is_running(procname, tries=3, wait=2):
-    """Given the name of a supervisord process, tell you whether it's running
-    or not.  If status is 'starting', will wait until status has settled."""
-    # Status return from supervisorctl will look like one of these:
-    # mysite_20110623_162319 RUNNING    pid 628, uptime 0:34:13
-    # mysite_20110623_231206 FATAL      Exited too quickly (process log may have details)
-
-    status_parts = sudo('supervisorctl status %s' % env.deployment).split()
-    if status_parts[1] == 'RUNNING':
-        return True
-    elif status_parts[1] == "FATAL":
-        return False
-    elif tries > 0:
-        # It's neither running nor dead yet, so try again
-        _yellow("Waiting %s seconds for process to settle" % wait)
-        time.sleep(wait)
-
-        # decrement the tries and double the wait time for the next check.
-        return _is_running(procname, tries - 1, wait * 2)
-    else:
-        return False
-
-def _is_this_site(name):
-    """Return True if 'name' matches our site name (old style Silk deployment
-    naming' or matches our name + timestamp pattern."""
-    
-    site_pattern = re.compile('%s_\d{8}_\d{6}' % env.site)
-    return (name == env.site) or (re.match(site_pattern, name) is not None)
+        if bad in txt:
+            txt = txt.replace(bad, good)
+            _write_file(file, txt, use_sudo=True, chown='root')
 
 def cleanup():
     """Deletes old versions of the site that are still sitting around."""
             if not _is_live(fullpath):
                 sudo('rm -rf %s' % fullpath)
 
-def start_supervisor():
+    # Clean up old socket files in /tmp/ that have no associated site
+    # TODO: use our own list dir function and a regular expression to filter
+    # the list of /tmp sockets instead of this funky grepping.
+    with cd('/tmp'):
+        socks = run('ls -1 | grep %s | grep sock | cat -' % env.site).split('\r\n')
+    for sock in socks:
+        procname = sock.replace('.sock', '')
+        if not exists(_join(SRV_ROOT, procname)):
+            sudo('rm /tmp/%s' % sock)
+
+    # Clean up supervisord includes that have no associated site.  There
+    # shouldn't be any since stop_other_processes handles those, but you never
+    # know.
+    configs = [x for x in _list_dir('/etc/supervisor/conf.d') if _is_this_site(x)]
+    for config in configs:
+        sudo('rm %s' % _join('/etc/supervisor/conf.d', config))
+
+    # TODO: clean out the pip-* folders that can build up in /tmp
+    # TODO: figure out a way to clean out pybundle files in /srv/_silk_build
+    # that aren't needed anymore.
+
+def start_process():
     """Tell supervisord to read the new config, then start the new process."""
     _green('STARTING PROCESS')
-    sudo('supervisorctl reread')
+    result = sudo('supervisorctl reread')
+
     sudo('supervisorctl add %s' % env.deployment)
 
 def _get_nginx_static_snippet(url_path, local_path):
 
     # Create nginx include here:
     # /etc/nginx/sites-enabled/<sitename>.conf
-    nginx_file = _join('/etc', 'nginx', 'sites-enabled', '%s.conf' % env.site)
-    sudo('rm %s' % nginx_file)
+    nginx_file = _join('/etc', 'nginx', 'sites-enabled', env.site)
+    sudo('rm -f %s' % nginx_file)
     _write_template('nginx.conf', nginx_file, template_vars)
 
 def switch_nginx():
     _green("LOADING NEW NGINX CONFIG")
 
-    # Check if there is an old-style version (withing the site root) of the
+    # Check if there is an old-style version (within the site root) of the
     # nginx config laying around, and rename it to something innocuous if so.
     old_nginx = _join(SRV_ROOT, env.site, 'conf', 'nginx.conf')
     if exists(old_nginx):
     configs."""
     proclist = sudo('supervisorctl status').split('\r\n')
 
+    # parse each line so we can get at just the proc names
+    proclist = [x.split() for x in proclist]
+
     # filter proclist to include only versions of our site
-    proclist = [x for x in proclist if _is_this_site(x)]
+    proclist = [x for x in proclist if _is_this_site(x[0])]
+
+    live_statuses = ["RUNNING", "STARTING"]
 
     # stop each process left in proclist that isn't the current one
     for proc in proclist:
         # We assume that spaces are not allowed in proc names
-        procparts = proc.split()
-        procname = procparts[0]
-        procstatus = procparts[1]
+        procname = proc[0]
+        procstatus = proc[1]
         if procname != env.deployment:
             # Stop the process
-            if procstatus == "RUNNING":
+            if procstatus in live_statuses:
                 sudo('supervisorctl stop %s' % procname)
 
             # Remove it from live config
             sudo('supervisorctl remove %s' % procname)
 
             # Remove its supervisord config file
-            conf_file = _join(SRV_ROOT, procname, 'conf', 'supervisord.conf')
+            conf_file = '/etc/supervisor/conf.d/%s.conf' % procname
             if exists(conf_file):
                 sudo('rm %s' % conf_file)
+
+            # Also remove old style supervisord include if it exists
+            old_conf_file = _join(SRV_ROOT, procname, 'conf/supervisord.conf')
+            if exists(old_conf_file):
+                sudo('rm %s' % old_conf_file)
+
     sudo('supervisorctl reread')
 
 def congrats():
 
 def push():
     """
-    The main function.  Assuming you have nginx and supervisord installed and
-    configured, this function will put your site on the remote host and get it
+    The main function.  This function will put your site on the remote host and get it
     running.
     """
+    # Make sure nginx and supervisord are installed
+    install_server_deps()
+
+    # Fix an embarrassing config bug from earlier versions
+    fix_supd_config_bug()
 
     # push site code and pybundle to server, in timestamped folder
     push_code()
     # make virtualenv on the server and run pip install on the pybundle
     create_virtualenv()
     install_bundle()
-    _write_blame()
+    _write_site_yaml()
     # write supervisord config for the new site
     configure_supervisor()
 
     ##then the magic
 
     ##silk starts up supervisord on the new site
-    start_supervisor()
-    # checks that the new site is running (by doing a supervisord xmlrpc
-    # request)
+    start_process()
+    # checks that the new site is running (by using supervisorctl) 
     if _is_running(env.deployment):
         _green("Site is running.  Proceeding with nginx switch.")
         # if the site's running fine, then silk configures nginx to forward requests
         congrats()
     else:
         _red("Process failed to start cleanly.  Off to the log files!")
+        sys.exit(1)
         else:
             return None
 
-def get_config(root=None, role=None):
-    """
-    Returns merged site and role config.  Tries hard to come up with
-    something if you don't pass in a role or site name.
-    """
-
-    # If no root given, then look above os.getcwd()
-    if root is None:
-        root = get_root(os.getcwd())
-
-    # If role is None, then try getting it from cmd line and/or env vars ourselves
-    role = role or get_role()
-
-    # If role is still none, then look for a blame file, which doesn't require a role.
-            # If no role and no blame file, give up.
-
-    config = get_site_config(root)
-
-    role_config = get_role_config(role)
-    if isinstance(role_config, dict):
-        config.update()
-    return config
-
 def get_template_path(template, root=None):
     """
     Returns path of template from site conf_templates dir, if found there, else

silk/sock_http.py

+#!/usr/bin/env python
+
+import sys
+import socket
+
+REQUEST_TEMPLATE = ('%(method)s %(path)s HTTP/1.1\r\n'
+                    'Host: %(host)s\r\n\r\n')
+
+SUPPORTED_METHODS = ('HEAD', 'GET')
+
+def sockhttp(sockpath, method, path, host):
+    """Make an HTTP request over a unix socket."""
+    req = REQUEST_TEMPLATE % locals()
+    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    s.connect(sockpath)
+    s.send(req)
+    out = ''
+    while 1:
+        data = s.recv(1024)
+        out += data
+        if not data: break
+    s.close()
+    return out
+
+def usage():
+    """Print usage information for this program"""
+    print ("This program allows you to make http requests to unix sockets. "
+           "Usage:\n\n"
+           "python %s /path/to/socket METHOD request_path host_name\n" %
+           __file__)
+
+    print "Supported methods are: %s" % ", ".join(SUPPORTED_METHODS)
+
+if __name__ == '__main__':
+    try:
+        _, sockpath, method, path, host = sys.argv
+    except ValueError:
+        usage()
+        sys.exit(1)
+    print sockhttp(sockpath, method, path, host)
+