1. Nate Aune
  2. silverlining

Commits

Ian Bicking  committed ff0c131

Added more formalized support for disabled applications, using 'silver disable'. From Peter Russell

  • Participants
  • Parent commits bec0a41
  • Branches default

Comments (0)

Files changed (14)

File docs/conf.py

View file
 
 html_theme = 'nature'
 html_theme_path = ['_theme']
-html_favicon = 'favicon.ico'
+#html_favicon = 'favicon.ico'
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths

File docs/disabling-sites.txt

View file
+.. -*- mode: rst -*-
+
+=================
+ Disabling Sites
+=================
+
+When hosting a site it's likely that from time to time it will be
+necessary to make it unavailable to the general public while some
+maintenance is carried out.  Silver Lining provides a mechanism for
+allowing this.  It is important to recognise that it is *applications*
+that are disabled via this mechanism, rather than locations, so if the
+same application is accessible via two different locations (even if it
+is two different *deployments* of the application), and it is
+disabled, then it will appear to be disabled at both locations.  This
+is because if your application uses a database, then all different
+deployments of the application will share the database, and cleaning
+up databases is one of the key use-cases for this feature.
+
+Disabling an application
+========================
+
+The following command will disable an application called blog::
+
+    $ silver disable --by-name blog
+
+You can also disable the application by giving a path that it appears
+at::
+
+    silver disable --by-location www.example.com/blog
+
+Enabling an application
+=======================
+
+To re-enable an application you use the ``silver enable`` command,
+which takes the same options as silver disable command.
+
+The disabled special location
+=============================
+When an application is disabled, visitors to it will be shown a page
+explaining that the site is unavailable (similarly there is a
+not-found location which is shown when a URL is not resolved).  It is
+possible to replace this page by uploading a new WSGI application to
+the special location "disabled".  To restore the default application,
+use the ``silver activate`` command to point the disabled location to
+"default-disabled".
+
+Headers for disabled apps
+-------------------------
+It is sensible to return a "503 Service Unavailable" response from a
+disabled app and use the following headers:
+
+:Cache-Control: no-store, no-cache, max-age=0
+:Pragma: no-cache
+
+Access To disabled apps
+=======================
+While an application is disabled, it is still possible to access it
+via the command line tools.  ``silver backup``, ``restore``, ``run``
+etc. will all work as usual.  It's also possible to access the site
+through a web browser.  At present the application is only made
+available to clients connecting from localhost, so to view the site
+and access management facilities you will probably need to set up an
+SSH tunnel and manually update ``/etc/hosts`` on your own machine.

File docs/index.txt

View file
     providers
     examples
     django-quickstart
+    disabling-sites
     docswanted
     todo
     comparisons

File silverlining/commands/disable.py

View file
+from cmdutils import CommandError
+from silversupport.appdata import normalize_location
+from silversupport.shell import ssh
+
+
+def command_disable(config, enable=False):
+    node, location, appname = None, None, None
+    if config.args.by_name is not None:
+        appname = config.args.by_name
+    if config.args.by_location is not None:
+        location = config.args.by_location
+        node, _path = normalize_location(location)
+    if config.args.node is not None:
+        node = config.node_hostname
+    if location and not appname:
+        stdout, _stderr, _returncode = ssh(
+            'www-mgr', node,
+            '/usr/local/share/silverlining/mgr-scripts/get-application.py',
+            fail_on_error=True)
+        appname = stdout.strip()
+    if node is None or appname is None:
+        raise CommandError("Unable to determine target node or appname.")
+    ssh('www-mgr', node,
+        '/usr/local/share/silverlining/mgr-scripts/disable.py %s %s'
+        % ('--enable' if enable else '', appname))

File silverlining/commands/enable.py

View file
+from silverlining.commands import disable
+
+
+def command_enable(config):
+    return disable.command_disable(config, enable=True)

File silverlining/commands/query.py

View file
 def command_query(config):
     stdout, stderr, returncode = ssh(
         'www-mgr', config.node_hostname,
-        'cat /var/www/appdata.map; echo "END" ; ls /var/www',
+        'cat /var/www/appdata.map; echo "END"; '
+        'cat /var/www/disabledapps.txt; echo END; '
+        'ls /var/www',
         capture_stdout=True)
     hosts = {}
     lines = [line.strip()
     site_instances = {}
     instance_site = {}
     sites = set()
+    disabled = set()
+
+    # parse appdata.map
     while 1:
         if not lines:
             break
             break
         hostname, path, data = line.split(None, 2)
         instance_name = data.split('|')[0]
-        hosts[hostname+path] = instance_name
+        hosts[hostname + path] = instance_name
+
+    # parse disabledsites.txt
+    while 1:
+        if not lines:
+            break
+        line = lines.pop(0)
+        if line == 'END':
+            break
+        disabled.add(line)
+
+    # parse directory listing
     for line in lines:
-        if line == 'appdata.map':
+        if line == 'appdata.map' or line == 'disabledapps.txt':
             continue
         match = re.match(r'^(?:([a-z0-9_.-]+)\.(.*)|default-[a-z]+)$',
                          line)
     for hostname, site in hosts.items():
         if site in ('disabled', 'notfound'):
             special_hosts.setdefault(site, []).append(hostname)
+        elif site in disabled:
+            special_hosts.setdefault('disabled', []).append(hostname)
     for site in sorted(sites):
         if len(sites) > 1:
             notify('Site: %s' % site)

File silverlining/mgr-scripts/disable.py

View file
+#!/usr/bin/env python
+
+import sys
+sys.path.insert(0, '/usr/local/share/silverlining/lib')
+import optparse
+from silversupport import disabledapps
+from silversupport import appdata
+
+parser = optparse.OptionParser(
+    usage='%prog application')
+
+parser.add_option(
+    '--enable',
+    action='store_true',
+    help='renable a previously disabled application')
+
+
+def main():
+    options, args = parser.parse_args()
+    application, = args
+    in_use = any(True for instance, uses in appdata.all_app_instances().items()
+                 if instance.startswith(application + '.') and uses)
+    if not in_use:
+        sys.stderr.write("%s is not in use in appdata.map\n" % application)
+        return 1
+    if options.enable:
+        if not disabledapps.is_disabled(application):
+            sys.stderr.write("%s is not disabled\n" % application)
+            return 1
+        disabledapps.enable_application(application)
+    else:
+        if disabledapps.is_disabled(application):
+            sys.stderr.write("%s is already disabled\n" % application)
+            return 1
+        disabledapps.disable_application(application)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

File silverlining/mgr-scripts/get-application.py

View file
+#!/usr/bin/env python
+"""Find the application for a location"""
+
+import sys
+sys.path.insert(0, '/usr/local/share/silverlining/lib')
+from silversupport import appdata
+
+
+def main():
+    hostname, path = appdata.normalize_location(sys.argv[1])
+    instance = appdata.instance_for_location(hostname, path)
+    appname = instance.split('.')[0]
+    print appname
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

File silverlining/runner.py

View file
     "they must all be on the same node.  Can be a wildcard.")
 
 parser_deactivate.add_argument(
-    '--disable', action='store_true',
-    help="Set the host to the status disabled, pointing it at the disabled application (good for a temporary removal)")
-
-parser_deactivate.add_argument(
     '--keep-prev', action='store_true',
     help="Keep the prev.* host activate (by default it is deleted)")
 
     '--info', action='store_true',
     help="Show information about how the configuration is created")
 
+parser_disable = subcommands.add_parser(
+    'disable', help="Temporarily disable an application")
+
+parser_enable = subcommands.add_parser(
+    'enable', help="Re-enable a disabled application")
+
+for subparser in (parser_disable, parser_enable):
+    group = subparser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--by-name', metavar="APPNAME",
+                       help="Identify the application by its name")
+    group.add_argument('--by-location', metavar="LOCATION",
+                       help="Identify the application by its location")
+    subparser.add_argument('--node', metavar="NODE_HOSTNAME",
+                           help="Node to act on")
+
 for subparser in subcommands._name_parser_map.values():
     subparser.add_argument(
         '-p', '--provider',

File silverlining/server-root/etc/varnish/default.vcl

View file
     remove req.http.X-Forwarded-For;
     set req.http.X-Forwarded-For = client.ip;
     if (req.request == "POST") {
-        pass;
+        return(pass);
     }
     if (req.request != "GET" && req.request != "HEAD" &&
         req.request != "PUT" && req.request != "POST" &&
         req.request != "DELETE") {
 
         # Non-RFC2616 or CONNECT which is weird. #
-        pass;
+        return(pass);
     }
     if (req.http.Authorization) {
         # Not cacheable by default #
-        pass;
+        return(pass);
     }
 }
 
 sub vcl_fetch {
-    if(obj.http.Pragma ~ "no-cache" ||
-       obj.http.Cache-Control ~ "no-cache" ||
-       obj.http.Cache-Control ~ "private") {
-            pass;
+    if(beresp.http.Pragma ~ "no-cache" ||
+       beresp.http.Cache-Control ~ "no-cache" ||
+       beresp.http.Cache-Control ~ "private") {
+            return(pass);
     }
-    if (obj.status >= 300) {
-        pass;
+    if (beresp.status >= 300) {
+        return(pass);
     }
     # Django regularly sends pages with Set-Cookie and cache control, 
     # we'll ignore Cache-Control in that case, as there's no point to
     # caching something that sets a cookie.
-    if (obj.http.Set-Cookie) {
-        unset obj.http.Cache-Control;
-        pass;
+    if (beresp.http.Set-Cookie) {
+        unset beresp.http.Cache-Control;
+        return(pass);
     }
-    if (obj.http.Cache-Control ~ "max-age" || obj.http.Expires) {
-        unset obj.http.Set-Cookie;
-        deliver;
+    if (beresp.http.Cache-Control ~ "max-age" || beresp.http.Expires) {
+        unset beresp.http.Set-Cookie;
+        return(deliver);
     }
-    pass;
+    return(pass);
 }
 
 sub vcl_hit {
     if (!obj.cacheable) {
-        pass;
+        return(pass);
     }
 
     if (req.http.Cache-Control ~ "no-cache") {
             return (restart);
         } 
     }
-    deliver;
+    return(deliver);
 }

File silversupport/appconfig.py

View file
 from silversupport.env import is_production
 from silversupport.shell import run
 from silversupport.util import asbool, read_config
+from silversupport.disabledapps import DisabledSite, is_disabled
 
 __all__ = ['AppConfig']
 
+DEPLOYMENT_LOCATION = "/var/www"
+
 
 class AppConfig(object):
     """This represents an application's configuration file, and by
     @classmethod
     def from_instance_name(cls, instance_name):
         """Loads an instance given its name; only valid in production"""
-        return cls(os.path.join('/var/www', instance_name, 'app.ini'))
+        return cls(os.path.join(DEPLOYMENT_LOCATION, instance_name, 'app.ini'))
 
     @classmethod
     def from_location(cls, location):
             execfile(sitecustomize, ns)
 
     def get_app_from_runner(self):
-        """Returns the WSGI app that the runner indicates"""
+        """Returns the WSGI app that the runner indicates
+        """
         assert self.platform == 'python', (
             "get_app_from_runner() shouldn't be run on an app with the platform %s"
             % self.platform)
             runner = 'config:%s' % runner
             global_conf = os.environ.copy()
             global_conf['SECRET'] = get_secret()
-            return loadapp(runner, name=spec,
-                           global_conf=global_conf)
+            app = loadapp(runner, name=spec,
+                          global_conf=global_conf)
         elif runner.endswith('.py'):
             ## FIXME: not sure what name to give it
             ns = {'__file__': runner, '__name__': 'main_py'}
             execfile(runner, ns)
             spec = spec or 'application'
             if spec in ns:
-                return ns[spec]
+                app = ns[spec]
             else:
                 raise Exception("No application %s defined in %s"
                                 % (spec, runner))
         else:
             raise Exception("Unknown kind of runner (%s)" % runner)
+        if is_production() and is_disabled(self.app_name):
+            disabled_appconfig = AppConfig.from_location('disabled')
+            return DisabledSite(app, disabled_appconfig.get_app_from_runner())
+        else:
+            return app
 
     def canonical_hostname(self):
         """Returns the 'canonical' hostname for this application.
         """Synchronize this application (locally) with a remote server
         at the given host.
         """
-        dest_dir = os.path.join('/var/www', instance_name)
+        dest_dir = os.path.join(DEPLOYMENT_LOCATION, instance_name)
         self._run_rsync(host, self.app_dir, dest_dir)
 
     def sync_config(self, host, config_dir):

File silversupport/disabledapps.py

View file
+"""Routines for managing the list of disabled applications"""
+
+# The file contains application names rather than instance names or locations,
+# as what we really care about is databases, and they are shared between
+# deployments of the same application.
+
+# The format of the disabled apps file is at present simply a list of
+# application names.  In the future it might well be desirable to extend this
+# with options for the DisabledSite WSGI middleware, such as a list of IPs
+# which can be allowed to access the site, or a set of http auth credentials
+# which allow access.  Happily the file should be empty except in exceptional
+# circumstances, and we can therefore just change the data format without
+# concern for backwards compatibility.
+
+import os.path
+
+
+DISABLED_APPS_FILE = '/var/www/disabledapps.txt'
+
+
+class DisabledSite(object):
+    """This WSGI app is returned instead of our application when it is disabled
+    """
+
+    def __init__(self, real_app, disabled_app):
+        self._app = real_app
+        self._disabled = disabled_app
+
+    def __call__(self, environ, start_response):
+        """Delegates to the disabled app unless some conditions are met"""
+        if environ.get('silverlining.update'):
+            return self._app(environ, start_response)
+        # Allow connections from localhost.
+        client = environ['REMOTE_ADDR']
+        if client.strip() in ('localhost', '127.0.0.1'):
+            return self._app(environ, start_response)
+        return self._disabled(environ, start_response)
+
+
+def is_disabled(application_name):
+    """Return True if the application has been disabled"""
+    if not os.path.exists(DISABLED_APPS_FILE):
+        return False
+    with open(DISABLED_APPS_FILE) as file_:
+        lines = [line.strip() for line in file_]
+        return application_name in lines
+
+
+def disable_application(application_name):
+    """Adds application_name to the list of disabled applications"""
+    if not is_disabled(application_name):
+        with open(DISABLED_APPS_FILE, 'a') as file_:
+            file_.write(application_name)
+            file_.write('\n')
+
+
+def enable_application(application_name):
+    """Removes application_name from the list of disabled applications"""
+    lines = []
+    with open(DISABLED_APPS_FILE, 'r') as file_:
+        for line in file_:
+            line = line.strip()
+            if line != application_name:
+                lines.append(line)
+    with open(DISABLED_APPS_FILE, 'w') as file_:
+        file_.write('\n'.join(lines))

File tests/unit/test_disabledapps.py

View file
+"""Tests for the functionality around disabled applications"""
+import os
+import os.path
+import tempfile
+from contextlib import contextmanager, nested
+import sys
+import shutil
+
+from webtest import TestApp
+
+from silversupport import disabledapps
+from silversupport.appconfig import AppConfig
+from silversupport import appdata
+
+ROOT = os.path.join(os.path.dirname(__file__),  '../..')
+
+
+@contextmanager
+def monkeypatch(module, global_, replacement):
+    """Replace module.global_ with replacement"""
+    if isinstance(module, str):
+        __import__(module)
+        module = sys.modules[module]
+    old = getattr(module, global_)
+    setattr(module, global_, replacement)
+    try:
+        yield
+    finally:
+        setattr(module, global_, old)
+
+
+@contextmanager
+def temporary_directory():
+    """Make and then remove a temporary directory"""
+    dirname = tempfile.mkdtemp()
+    try:
+        yield dirname
+    finally:
+        shutil.rmtree(dirname)
+
+
+@contextmanager
+def patch_disabled_apps_path(dirname):
+    """Changes the location of the disabled apps list to be within dirname
+
+    Returns the new path of the file
+    """
+    with monkeypatch(disabledapps, 'DISABLED_APPS_FILE',
+                     os.path.join(dirname, 'disabledapps.txt')):
+        try:
+            yield disabledapps.DISABLED_APPS_FILE
+        finally:
+            if os.path.exists(disabledapps.DISABLED_APPS_FILE):
+                os.remove(disabledapps.DISABLED_APPS_FILE)
+
+
+def test_addition():
+    """Adding an application should correctly write the file"""
+    with temporary_directory() as tempdir:
+        with patch_disabled_apps_path(tempdir) as path:
+            disabledapps.disable_application('testapp')
+            with open(path) as file_:
+                lines = [line.strip() for line in file_]
+                assert lines == ['testapp'], lines
+
+
+def test_is_disabled():
+    """We should be able to identify disabled apps"""
+    with temporary_directory() as tempdir:
+        with patch_disabled_apps_path(tempdir):
+            assert not disabledapps.is_disabled('testapp')
+            disabledapps.disable_application('testapp')
+            assert disabledapps.is_disabled('testapp')
+
+
+def test_removal():
+    """Removing applications should work correctly"""
+    with temporary_directory() as tempdir:
+        with patch_disabled_apps_path(tempdir) as path:
+            disabledapps.disable_application('testapp-a')
+            disabledapps.disable_application('testapp-b')
+            disabledapps.disable_application('testapp-c')
+            disabledapps.enable_application('testapp-b')
+            with open(path) as file_:
+                lines = [line.strip() for line in file_]
+                assert lines == ['testapp-a', 'testapp-c'], lines
+
+
+## Utilities to help set up a disabled site
+def patch_deployment_location(dirname):
+    """Replace /var/www with dirname, and set is_production to True"""
+    return nested(
+        patch_disabled_apps_path(dirname),
+        monkeypatch(appdata, 'APPDATA_MAP',
+                    os.path.join(dirname, 'appdata.map')),
+        monkeypatch('silversupport.appconfig', 'DEPLOYMENT_LOCATION', dirname),
+        monkeypatch('silversupport.appconfig', 'is_production', lambda: True))
+
+
+def install_default_disabled(dirname):
+    """Copy default-disabled into dirname"""
+    default_disabled = os.path.join(
+        ROOT, 'silverlining/server-root/var/www/default-disabled')
+    shutil.copytree(default_disabled,
+                    os.path.join(dirname, 'default-disabled'))
+    with open(os.path.join(dirname, 'appdata.map'), 'w') as file_:
+        file_.write(
+            "disabled / default-disabled|general_debug|/dev/null|python|\n")
+
+
+def install_sample_app(dirname):
+    """copy sample app into dirname"""
+    path = os.path.join(dirname, 'sampleapp.0')
+    os.mkdir(path)
+    with open(os.path.join(path, 'app.ini'), 'w') as file_:
+        file_.write('[production]\n')
+        file_.write('app_name = sampleapp\n')
+        file_.write('runner = app.py#application\n')
+    with open(os.path.join(path, 'app.py'), 'w') as file_:
+        file_.write(
+            'from wsgiref.simple_server import demo_app as application\n')
+    appdata.add_appdata('sampleapp.0', ['www.example.com'])
+
+
+@contextmanager
+def disabled_site():
+    with temporary_directory() as tempdir:
+        with patch_deployment_location(tempdir):
+
+            install_default_disabled(tempdir)
+            install_sample_app(tempdir)
+            disabledapps.disable_application('sampleapp')
+
+            app_config = AppConfig.from_location('www.example.com')
+            app = app_config.get_app_from_runner()
+            yield app
+
+
+def test_loading_disabled_site():
+    """Loading a site should return a DisabledSite when the site is disabled"""
+    with disabled_site() as app:
+        assert isinstance(app, disabledapps.DisabledSite), app
+
+
+def test_disabled_site_is_disabled():
+    with disabled_site() as app:
+        test_app = TestApp(app)
+        response = test_app.get(
+            '/', headers={'X-Forwarded-For':'123.123.123.123, localhost'},
+            status=503)
+
+
+def test_disabled_site_commandline_internal_access():
+    with disabled_site() as app:
+        test_app = TestApp(app)
+        response = test_app.get(
+            '/', headers={'X-Forwarded-For':'123.123.123.123, localhost'},
+            extra_environ={'silverlining.update': True},
+            status=200)
+        assert response.body.startswith('Hello world!'), str(response)
+
+
+def test_disabled_site_local_access():
+    with disabled_site() as app:
+        test_app = TestApp(app)
+        response = test_app.get(
+            '/', headers={'X-Forwarded-For':'localhost, localhost'})
+        assert response.status.startswith('200')
+        assert response.body.startswith('Hello world!')

File tests/unit/test_rewrite_map.py

View file
     for line in tests:
         input, output = line.split(None, 1)
         hostname, path = input.split('^')
-        path_match, data = rewritemap.lookup(hostname, path)
+        path_match, data, _rest = rewritemap.lookup(hostname, path)
         result = '%s %s' % (path_match, data)
         assert result == output, (
             "Bad result: %r -> %r" % (line, result))